Add a first pass at the HTTP client API

Author: pimterry

package-lock.json

@@ -98,7 +98,7 @@
     "@types/klaw": "^3.0.2",
     "@types/lodash": "^4.14.117",
     "@types/mocha": "^5.2.5",
-    "@types/node": "^16.3.2",
+    "@types/node": "^16.18.38",
     "@types/node-forge": "^0.9.9",
     "@types/request-promise-native": "^1.0.15",
     "@types/rimraf": "^2.0.2",

package.json

@@ -10,6 +10,7 @@ import { reportError, addBreadcrumb } from '../error-tracking';
 import { HtkConfig } from "../config";
 import { ActivationError, Interceptor } from "../interceptors";
 import { getDnsServer } from '../dns-server';
+import * as Client from '../client/client';
 
 const INTERCEPTOR_TIMEOUT = 1000;
 
@@ -194,6 +195,13 @@ export class ApiModel {
         return { success: !interceptor.isActive(proxyPort) };
     }
 
+    async sendRequest(
+        requestDefinition: Client.RequestDefinition,
+        requestOptions: Client.RequestOptions
+    ): Promise<Client.ResponseDefinition> {
+        return Client.sendRequest(requestDefinition, requestOptions);
+    }
+
 }
 
 // Wait for a promise, falling back to defaultValue on error or timeout

src/api/api-model.ts

@@ -12,6 +12,7 @@ import type { ParsedQs } from 'qs';
 
 import { ErrorLike, StatusError } from '../util/error';
 import { ApiModel } from './api-model';
+import * as Client from '../client/client';
 
 /**
  * This file exposes the API model via a REST-ish classic HTTP API.
@@ -73,6 +74,34 @@ export function exposeRestAPI(
         const result = await apiModel.activateInterceptor(interceptorId, proxyPort, interceptorOptions);
         res.send({ result });
     }));
+
+    server.post('/client/send', handleErrors(async (req, res) => {
+        const bodyData = req.body;
+        if (!bodyData) throw new StatusError(400, "No request definition or options provided");
+
+        const {
+            request,
+            options
+        } = bodyData;
+
+        if (!request) throw new StatusError(400, "No request definition provided");
+        if (!options) throw new StatusError(400, "No request options provided");
+
+        const result = await apiModel.sendRequest({
+            ...request,
+            // Body buffers are serialized as base64 (for both requests & responses)
+            rawBody: Buffer.from(request.rawBody ?? '', 'base64')
+        }, {
+            ...options
+        });
+
+        res.send({
+            result: {
+                ...result,
+                rawBody: result.rawBody?.toString('base64') ?? ''
+            }
+        });
+    }));
 }
 
 function getProxyPort(stringishInput: any) {

src/api/rest-api.ts

@@ -0,0 +1,83 @@
+import * as net from 'net';
+import * as http from 'http';
+import * as https from 'https';
+import { streamToBuffer } from '../util/stream';
+
+export type RawHeaders = Array<[key: string, value: string]>;
+
+export interface RequestDefinition {
+    method: string;
+    url: string;
+    headers: RawHeaders;
+    rawBody?: Uint8Array;
+}
+
+export interface RequestOptions {
+}
+
+export interface ResponseDefinition {
+    statusCode: number;
+    statusMessage?: string;
+    headers: RawHeaders;
+    rawBody?: Buffer;
+}
+
+export async function sendRequest(
+    requestDefn: RequestDefinition,
+    options: RequestOptions
+): Promise<ResponseDefinition> {
+    const url = new URL(requestDefn.url);
+
+    const request = (url.protocol === 'https:' ? https : http).request({
+        protocol: url.protocol,
+        host: url.host,
+        port: url.port,
+        path: url.pathname,
+
+        method: requestDefn.method,
+
+        // Node supports sending raw headers via [key, value, key, value] array, but we need an
+        // 'any' as the types don't believe it:
+        headers: flattenPairedRawHeaders(requestDefn.headers) as any
+    });
+
+    if (requestDefn.rawBody?.byteLength) {
+        request.end(requestDefn.rawBody);
+    } else {
+        request.end();
+    }
+
+    const response = await new Promise<http.IncomingMessage>((resolve, reject) => {
+        request.on('error', reject);
+        request.on('response', resolve);
+    });
+
+    const body = await streamToBuffer(response);
+
+    return {
+        statusCode: response.statusCode!,
+        statusMessage: response.statusMessage,
+        headers: pairFlatRawHeaders(response.rawHeaders),
+        rawBody: body
+    }
+}
+
+/**
+ * Turn node's _very_ raw headers ([k, v, k, v, ...]) into our slightly more convenient
+ * pairwise tuples [[k, v], [k, v], ...] RawHeaders structure.
+ */
+export function pairFlatRawHeaders(flatRawHeaders: string[]): RawHeaders {
+    const result: RawHeaders = [];
+    for (let i = 0; i < flatRawHeaders.length; i += 2 /* Move two at a time */) {
+        result[i/2] = [flatRawHeaders[i], flatRawHeaders[i+1]];
+    }
+    return result;
+}
+
+/**
+ * Turn our raw headers [[k, v], [k, v], ...] tuples into Node's very flat
+ * [k, v, k, v, ...] structure.
+ */
+export function flattenPairedRawHeaders(rawHeaders: RawHeaders): string[] {
+    return rawHeaders.flat();
+}