Use metadata from HTTP proxy auth, unwrapping proxy-* headers up front

If you authenticate to Mockttp with username 'metadata' and a JSON (raw
of b64url encoded) password, that password will be used as metadata for
the request. Most notably, the "tags" field will be directly attached to
all request events later on. This applies to direct HTTP proxy
connections (GET http://example.com) and CONNECT-tunnelled connections
for both HTTP/1 & HTTP/2. This roughly matches the behaviour available
for SOCKS tunnels, which already support similar metadata.

Previously, proxy-connection headers were manually dropped from data
during rule processing. Now both proxy-connection & proxy-authorization
are preprocessed up front when sent with an absolute URL (i.e. when
a client is making a non-CONNECT tunnel) so they never appear in events
or traffic elsewhere.

Author: pimterry

src/rules/requests/request-handlers.ts

@@ -737,13 +737,6 @@ export class PassThroughHandler extends PassThroughHandlerDefinition {
             rawHeaders = h2HeadersToH1(rawHeaders);
         }
 
-        // Drop proxy-connection header. This is almost always intended for us, not for upstream servers,
-        // and forwarding it causes problems (most notably, it triggers lots of weird-traffic blocks,
-        // most notably by Cloudflare).
-        rawHeaders = rawHeaders.filter(([key]) =>
-            key.toLowerCase() !== 'proxy-connection'
-        );
-
         let serverReq: http.ClientRequest;
         return new Promise<void>((resolve, reject) => (async () => { // Wrapped to easily catch (a)sync errors
             serverReq = await makeRequest({

src/rules/websockets/websocket-handlers.ts

@@ -10,7 +10,7 @@ import {
     deserializeProxyConfig
 } from "../../serialization/serialization";
 
-import { Headers, OngoingRequest, RawHeaders } from "../../types";
+import { OngoingRequest, RawHeaders } from "../../types";
 
 import {
     CloseConnectionHandler,
@@ -19,6 +19,8 @@ import {
     TimeoutHandler
 } from '../requests/request-handlers';
 import { getEffectivePort } from '../../util/url';
+import { resetOrDestroy } from '../../util/socket-util';
+import { LastHopEncrypted } from '../../util/socket-extensions';
 import { isHttp2 } from '../../util/request-utils';
 import {
     findRawHeader,
@@ -27,7 +29,6 @@ import {
     pairFlatRawHeaders,
     rawHeadersToObjectPreservingCase
 } from '../../util/header-utils';
-import { streamToBuffer } from '../../util/buffer-utils';
 import { MaybePromise } from '@httptoolkit/util';
 
 import { getAgent } from '../http-agents';
@@ -51,7 +52,6 @@ import {
     WebSocketHandlerDefinition,
     WsHandlerDefinitionLookup,
 } from './websocket-handler-definitions';
-import { LastHopEncrypted, resetOrDestroy } from '../../util/socket-util';
 
 export interface WebSocketHandler extends WebSocketHandlerDefinition {
     handle(

src/server/http-combo-server.ts

@@ -25,6 +25,9 @@ import {
     getParentSocket,
     buildSocketTimingInfo,
     buildTlsSocketEventData,
+    resetOrDestroy
+} from '../util/socket-util';
+import {
     SocketIsh,
     InitialRemoteAddress,
     InitialRemotePort,
@@ -34,10 +37,10 @@ import {
     TlsMetadata,
     TlsSetupCompleted,
     SocketMetadata,
-    resetOrDestroy
-} from '../util/socket-util';
+} from '../util/socket-extensions';
 import { MockttpHttpsOptions } from '../mockttp';
 import { buildSocksServer, SocksServerOptions, SocksTcpAddress } from './socks-server';
+import { getSocketMetadataFromProxyAuth } from '../util/socket-metadata';
 
 // Hardcore monkey-patching: force TLSSocket to link servername & remoteAddress to
 // sockets as soon as they're available, without waiting for the handshake to fully
@@ -360,6 +363,9 @@ export async function createComboServer(options: ComboServerOptions): Promise<De
         socket.write('HTTP/' + req.httpVersion + ' 200 OK\r\n\r\n', 'utf-8', () => {
             socket[SocketTimingInfo]!.tunnelSetupTimestamp = now();
             socket[LastTunnelAddress] = connectUrl;
+            if (req.headers['proxy-authorization']) {
+                socket[SocketMetadata] = getSocketMetadataFromProxyAuth(socket, req.headers['proxy-authorization']);
+            }
             server.emit('connection', socket);
         });
     }
@@ -378,8 +384,12 @@ export async function createComboServer(options: ComboServerOptions): Promise<De
 
         // Send a 200 OK response, and start the tunnel:
         res.writeHead(200, {});
+
         inheritSocketDetails(res.socket, res.stream);
         res.stream[LastTunnelAddress] = connectUrl;
+        if (req.headers['proxy-authorization']) {
+            res.stream[SocketMetadata] = getSocketMetadataFromProxyAuth(res.stream, req.headers['proxy-authorization']);
+        }
 
         // When layering HTTP/2 on JS streams, we have to make sure the JS stream won't autoclose
         // when the other side does, because the upper HTTP/2 layers want to handle shutdown, so

src/server/mockttp-server.ts

@@ -59,14 +59,17 @@ import {
 import {
     buildRawSocketEventData,
     buildTlsSocketEventData,
+    isSocketLoop,
+    resetOrDestroy
+} from "../util/socket-util";
+import {
     ClientErrorInProgress,
     LastHopEncrypted,
     LastTunnelAddress,
     TlsSetupCompleted,
-    isSocketLoop,
-    resetOrDestroy,
-    getSocketMetadataTags
-} from "../util/socket-util";
+    SocketMetadata
+} from '../util/socket-extensions';
+import { getSocketMetadataTags, getSocketMetadataFromProxyAuth } from '../util/socket-metadata'
 import {
     parseRequestBody,
     waitForCompletedRequest,
@@ -93,6 +96,7 @@ type ExtendedRawRequest = (http.IncomingMessage | http2.Http2ServerRequest) & {
     body?: OngoingBody;
     path?: string;
     destination?: Destination;
+    [SocketMetadata]?: SocketMetadata;
 };
 
 const serverPortCheckMutex = new Mutex();
@@ -600,10 +604,13 @@ export class MockttpServer extends AbstractMockttp implements Mockttp {
     private preprocessRequest(req: ExtendedRawRequest, type: 'request' | 'websocket'): OngoingRequest {
         parseRequestBody(req, { maxSize: this.maxBodySize });
 
+        let rawHeaders = pairFlatRawHeaders(req.rawHeaders);
+        let socketMetadata: SocketMetadata | undefined = req.socket[SocketMetadata];
+
         // Make req.url always absolute, if it isn't already, using the host header.
         // It might not be if this is a direct request, or if it's being transparently proxied.
         if (!isAbsoluteUrl(req.url!)) {
-            req.protocol = req.headers[':scheme'] as string ||
+            req.protocol = getHeaderValue(rawHeaders, ':scheme') ||
                 (req.socket[LastHopEncrypted] ? 'https' : 'http');
             req.path = req.url;
 
@@ -617,8 +624,8 @@ export class MockttpServer extends AbstractMockttp implements Mockttp {
             // If you explicitly tunnel to a hostname, that's the URL's hostname:
             const hostname = tunnelUrlHost
                 // Otherwise, we infer based on headers: HTTP/2 or HTTP/1
-                ?? getHeaderValue(req.headers, ':authority')
-                ?? getHeaderValue(req.headers, 'host')
+                ?? getHeaderValue(rawHeaders, ':authority')
+                ?? getHeaderValue(rawHeaders, 'host')
                 ?? req.socket[LastTunnelAddress] // Iff we have no hostname available at all
                 ?? `localhost:${this.port}`; // If you specify literally nothing, it's a direct request
 
@@ -637,7 +644,7 @@ export class MockttpServer extends AbstractMockttp implements Mockttp {
 
             const absoluteUrl = `${req.protocol}://${host}${req.path}`;
 
-            if (!req.headers[':path']) {
+            if (!getHeaderValue(rawHeaders, ':path')) {
                 (req as Mutable<ExtendedRawRequest>).url = new url.URL(absoluteUrl).toString();
             } else {
                 // Node's HTTP/2 compat logic maps .url to headers[':path']. We want them to
@@ -648,12 +655,27 @@ export class MockttpServer extends AbstractMockttp implements Mockttp {
                 });
             }
         } else {
+            // We have an absolute request. This is effectively a combined tunnel + end-server request,
+            // so we need to handle both of those, and hide the proxy-specific bits from later logic.
             req.protocol = req.url!.split('://', 1)[0];
             req.path = getPathFromAbsoluteUrl(req.url!);
             req.destination = getDestination(
                 req.protocol,
                 req.socket[LastTunnelAddress] ?? getHostFromAbsoluteUrl(req.url!)
             );
+
+            const proxyAuthHeader = getHeaderValue(rawHeaders, 'proxy-authorization');
+            if (proxyAuthHeader) {
+                // Use this metadata for this request, but _only_ this request - it's not relevant
+                // to other requests on the same socket so we don't add it to req.socket.
+                socketMetadata = getSocketMetadataFromProxyAuth(req.socket, proxyAuthHeader);
+            }
+
+            rawHeaders = rawHeaders.filter(([key]) => {
+                const lcKey = key.toLowerCase();
+                return lcKey !== 'proxy-connection' &&
+                    lcKey !== 'proxy-authorization';
+            })
         }
 
         if (type === 'websocket') {
@@ -668,7 +690,8 @@ export class MockttpServer extends AbstractMockttp implements Mockttp {
         }
 
         const id = uuid();
-        const tags: string[] = getSocketMetadataTags(req.socket);
+
+        const tags: string[] = getSocketMetadataTags(socketMetadata);
 
         const timingEvents: TimingEvents = {
             startTime: Date.now(),
@@ -679,7 +702,6 @@ export class MockttpServer extends AbstractMockttp implements Mockttp {
             timingEvents.bodyReceivedTimestamp ||= now();
         });
 
-        const rawHeaders = pairFlatRawHeaders(req.rawHeaders);
         const headers = rawHeadersToObject(rawHeaders);
 
         // Not writable for HTTP/2:
@@ -1027,7 +1049,7 @@ ${await this.suggestRule(request)}`
                 id: uuid(),
                 tags: [
                     `client-error:${error.code || 'UNKNOWN'}`,
-                    ...getSocketMetadataTags(socket)
+                    ...getSocketMetadataTags(socket[SocketMetadata])
                 ],
                 timingEvents: { startTime: Date.now(), startTimestamp: now() } as TimingEvents
             };
@@ -1120,7 +1142,7 @@ ${await this.suggestRule(request)}`
                 tags: [
                     `client-error:${error.code || 'UNKNOWN'}`,
                     ...(isBadPreface ? ['client-error:bad-preface'] : []),
-                    ...getSocketMetadataTags(socket)
+                    ...getSocketMetadataTags(socket?.[SocketMetadata])
                 ],
                 httpVersion: '2',
 

src/server/socks-server.ts

@@ -1,6 +1,9 @@
 import * as _ from 'lodash';
 import * as net from 'net';
-import { resetOrDestroy, SocketMetadata } from '../util/socket-util';
+
+import { resetOrDestroy } from '../util/socket-util';
+import { SocketMetadata } from '../util/socket-extensions';
+import { getSocketMetadata } from '../util/socket-metadata';
 
 export interface SocksServerOptions {
     /**
@@ -259,9 +262,8 @@ async function handleCustomMetadata(socket: net.Socket) {
     const metadata = await readBytes(socket, length);
     const metadataString = metadata.toString('utf8');
 
-    let metadataJson: any = {};
     try {
-        metadataJson = JSON.parse(metadataString);
+        socket[SocketMetadata] = getSocketMetadata(socket[SocketMetadata], metadataString);
     } catch (e) {
         const errorData = Buffer.from(JSON.stringify({ message: 'Invalid JSON' }));
         const errorResponse = Buffer.alloc(4 + errorData.byteLength);
@@ -272,7 +274,7 @@ async function handleCustomMetadata(socket: net.Socket) {
         socket.end(errorResponse);
         return false;
     }
-    socket[SocketMetadata] = _.merge(socket[SocketMetadata] || {}, metadataJson);
+
     socket.write(Buffer.from([
         0x05, // Version
         0x00 // Success
@@ -296,15 +298,8 @@ async function handleUsernamePasswordMetadata(socket: net.Socket) {
         return false;
     }
 
-    let metadataJson: any = {};
     try {
-        // Base64'd json always starts with 'e' (typically eyI), so we can use this fairly
-        // reliably to detect base64 (and definitely exclude valid object JSON encoding).
-        const decoded = password[0] === 'e'.charCodeAt(0)
-            ? Buffer.from(password.toString('utf8'), 'base64url').toString('utf8')
-            : password.toString('utf8');
-
-        metadataJson = JSON.parse(decoded);
+        socket[SocketMetadata] = getSocketMetadata(socket[SocketMetadata], password);
     } catch (e) {
         socket.end(Buffer.from([
             0x05,
@@ -313,7 +308,6 @@ async function handleUsernamePasswordMetadata(socket: net.Socket) {
         return false;
     }
 
-    socket[SocketMetadata] = _.merge(socket[SocketMetadata] || {}, metadataJson);
     socket.write(Buffer.from([
         0x05, // Version
         0x00 // Success

src/types.ts

@@ -1,6 +1,6 @@
-import stream = require('stream');
-import http = require('http');
-import { EventEmitter } from 'events';
+import type * as stream from 'stream';
+import type * as http from 'http';
+import type { EventEmitter } from 'events';
 
 export const DEFAULT_ADMIN_SERVER_PORT = 45454;
 

src/util/request-utils.ts

@@ -40,13 +40,12 @@ import {
     pairFlatRawHeaders,
     rawHeadersToObject
 } from './header-utils';
-import { LastHopEncrypted, LastTunnelAddress } from './socket-util';
+import { LastHopEncrypted, LastTunnelAddress } from './socket-extensions';
 import { getDestination, normalizeHost } from './url';
 
 export const shouldKeepAlive = (req: OngoingRequest): boolean =>
     req.httpVersion !== '1.0' &&
-    req.headers['connection'] !== 'close' &&
-    req.headers['proxy-connection'] !== 'close';
+    req.headers['connection'] !== 'close';
 
 export const writeHead = (
     response: http.ServerResponse | http2.Http2ServerResponse,