Add support for 'additionalTrustedCAs' option in passthrough proxy config

Author: pimterry

src/main.ts

@@ -57,6 +57,7 @@ export type {
     ProxySettingCallbackParams
 } from './rules/proxy-config';
 export type {
+    CADefinition,
     ForwardingOptions,
     PassThroughLookupOptions,
     PassThroughHandlerConnectionOptions

src/rules/http-agents.ts

@@ -12,6 +12,7 @@ const getSocksProxyAgent = (opts: any) => new SocksProxyAgent(opts);
 
 import { isNode } from "../util/util";
 import { getProxySetting, matchesNoProxy, ProxySettingSource } from './proxy-config';
+import { getTrustedCAs } from './passthrough-handling';
 
 const KeepAliveAgents = isNode
     ? { // These are only used (and only available) on the node server side
@@ -70,24 +71,30 @@ export async function getAgent({
 
             const cacheKey = getCacheKey({
                 url: proxySetting.proxyUrl,
-                ca: proxySetting.trustedCAs
+                trustedCAs: proxySetting.trustedCAs,
+                additionalTrustedCAs: proxySetting.additionalTrustedCAs
             });
 
             if (!proxyAgentCache.has(cacheKey)) {
                 const { protocol, auth, hostname, port } = url.parse(proxySetting.proxyUrl);
                 const buildProxyAgent = ProxyAgentFactoryMap[protocol as keyof typeof ProxyAgentFactoryMap];
 
+                // If you specify trusted CAs, we override the CAs used for this connection, i.e. the trusted
+                // CA for the certificate of an HTTPS proxy. This is *not* the CAs trusted for upstream servers
+                // on the otherside of the proxy - see the corresponding passthrough options for that.
+                const trustedCerts = await getTrustedCAs(
+                    proxySetting.trustedCAs,
+                    proxySetting.additionalTrustedCAs
+                );
+
                 proxyAgentCache.set(cacheKey, buildProxyAgent({
                     protocol,
                     auth,
                     hostname,
                     port,
 
-                    // If you specify trusted CAs, we override the CAs used for this connection, i.e. the trusted
-                    // CA for the certificate of an HTTPS proxy. This is *not* the CAs trusted for upstream servers
-                    // on the otherside of the proxy - see the `trustAdditionalCAs` passthrough option for that.
-                    ...(proxySetting.trustedCAs
-                        ? { ca: proxySetting.trustedCAs }
+                    ...(trustedCerts
+                        ? { ca: trustedCerts }
                         : {}
                     )
                 }));

src/rules/passthrough-handling-definitions.ts

@@ -28,6 +28,10 @@ export interface PassThroughLookupOptions {
     servers?: string[];
 }
 
+export type CADefinition =
+    | { cert: string | Buffer }
+    | { certPath: string };
+
 /**
  * This defines the upstream connection parameters. These passthrough parameters
  * are shared between both WebSocket & Request passthrough rules.
@@ -62,7 +66,7 @@ export interface PassThroughHandlerConnectionOptions {
      * or buffer value containing the PEM certificate, or a `certPath` key and a
      * string value containing the local path to the PEM certificate.
      */
-    trustAdditionalCAs?: Array<{ cert: string | Buffer } | { certPath: string }>;
+    trustAdditionalCAs?: Array<CADefinition>;
 
     /**
      * A mapping of hosts to client certificates to use, in the form of

src/rules/passthrough-handling.ts

@@ -1,4 +1,5 @@
 import * as _ from 'lodash';
+import * as fs from 'fs/promises';
 import * as tls from 'tls';
 import url = require('url');
 import { oneLine } from 'common-tags';
@@ -17,6 +18,7 @@ import {
     CallbackResponseMessageResult
 } from './requests/request-handler-definitions';
 import {
+    CADefinition,
     PassThroughLookupOptions
 } from './passthrough-handling-definitions';
 
@@ -97,6 +99,34 @@ export const getUpstreamTlsOptions = (strictChecks: boolean): tls.SecureContextO
     rejectUnauthorized: strictChecks,
 });
 
+export async function getTrustedCAs(
+    trustedCAs: Array<string | CADefinition> | undefined,
+    additionalTrustedCAs: Array<CADefinition> | undefined
+): Promise<Array<string> | undefined> {
+    if (trustedCAs && additionalTrustedCAs) {
+        throw new Error(`trustedCAs and additionalTrustedCAs options are mutually exclusive`);
+    }
+
+    if (trustedCAs) {
+        return Promise.all(trustedCAs.map((caDefinition) =>  getCA(caDefinition)));
+    }
+
+    if (additionalTrustedCAs) {
+        const CAs = await Promise.all(additionalTrustedCAs.map((caDefinition) =>  getCA(caDefinition)));
+        return tls.rootCertificates.concat(CAs);
+    }
+}
+
+const getCA = async (caDefinition: string | CADefinition) => {
+    return typeof caDefinition === 'string'
+        ? caDefinition
+    : 'certPath' in caDefinition
+        ? await fs.readFile(caDefinition.certPath, 'utf8')
+    // 'cert' in caDefinition
+        : caDefinition.cert.toString('utf8')
+}
+
+
 // --- Various helpers for deriving parts of request/response data given partial overrides: ---
 
 /**

src/rules/proxy-config.ts

@@ -2,6 +2,7 @@ import * as _ from 'lodash';
 
 import { MaybePromise } from '../util/type-utils';
 import { RuleParameterReference } from './rule-parameters';
+import { CADefinition } from './passthrough-handling-definitions';
 
 /**
  * A ProxySetting is a specific proxy setting to use, which is passed to a proxy agent
@@ -41,10 +42,27 @@ export interface ProxySetting {
      * the proxy is not HTTPS. If not specified, this will default to the Node
      * defaults, or you can override them here completely.
      *
-     * Note that unlike passthrough rule's `trustAdditionalCAs` option, this sets the
-     * complete list of trusted CAs - not just additional ones.
+     * This sets the complete list of trusted CAs, and is mutually exclusive with the
+     * `additionalTrustedCAs` option, which adds additional CAs (but also trusts the
+     * Node default CAs too).
+     *
+     * This should be specified as either a { cert: string | Buffer } object or a
+     * { certPath: string } object (to read the cert from disk). The previous
+     * simple string format is supported but deprecated.
+     */
+    trustedCAs?: Array<
+        | string // Deprecated
+        | CADefinition
+    >;
+
+    /**
+     * Extra CAs to trust for HTTPS connections to the proxy. Ignored if the connection
+     * to the proxy is not HTTPS.
+     *
+     * This appends to the list of trusted CAs, and is mutually exclusive with the
+     * `trustedCAs` option, which completely overrides the list of CAs.
      */
-    trustedCAs?: string[];
+    additionalTrustedCAs?: Array<CADefinition>;
 }
 
 /**

src/rules/requests/request-handlers.ts

@@ -78,7 +78,8 @@ import {
     getUpstreamTlsOptions,
     shouldUseStrictHttps,
     getClientRelativeHostname,
-    getDnsLookupFunction
+    getDnsLookupFunction,
+    getTrustedCAs
 } from '../passthrough-handling';
 
 import {
@@ -388,16 +389,7 @@ export class PassThroughHandler extends PassThroughHandlerDefinition {
         if (!this.extraCACertificates.length) return undefined;
 
         if (!this._trustedCACertificates) {
-            this._trustedCACertificates = Promise.all(
-                (tls.rootCertificates as Array<string | Promise<string>>)
-                    .concat(this.extraCACertificates.map(certObject => {
-                        if ('cert' in certObject) {
-                            return certObject.cert.toString('utf8');
-                        } else {
-                            return fs.readFile(certObject.certPath, 'utf8');
-                        }
-                    }))
-            );
+            this._trustedCACertificates = getTrustedCAs(undefined, this.extraCACertificates);
         }
 
         return this._trustedCACertificates;

src/serialization/serialization.ts

@@ -286,13 +286,9 @@ export function deserializeBuffer(buffer: string): Buffer {
 const SERIALIZED_PARAM_REFERENCE = "__mockttp__param__reference__";
 export type SerializedRuleParameterReference<R> = { [SERIALIZED_PARAM_REFERENCE]: string };
 
-export function maybeSerializeParam<T, R>(value: T | RuleParameterReference<R>): T | SerializedRuleParameterReference<R> {
-    if (isParamReference(value)) {
-        // Swap the symbol for a string, since we can't serialize symbols in JSON:
-        return { [SERIALIZED_PARAM_REFERENCE]: value[MOCKTTP_PARAM_REF] };
-    } else {
-        return value;
-    }
+function serializeParam<R>(value: RuleParameterReference<R>): SerializedRuleParameterReference<R> {
+    // Swap the symbol for a string, since we can't serialize symbols in JSON:
+    return { [SERIALIZED_PARAM_REFERENCE]: value[MOCKTTP_PARAM_REF] };
 }
 
 function isSerializedRuleParam(value: any): value is SerializedRuleParameterReference<unknown> {
@@ -335,8 +331,22 @@ export function serializeProxyConfig(
         return callbackId;
     } else if (_.isArray(proxyConfig)) {
         return proxyConfig.map((config) => serializeProxyConfig(config, channel));
-    } else {
-        return maybeSerializeParam(proxyConfig);
+    } else if (isParamReference(proxyConfig)) {
+        return serializeParam(proxyConfig);
+    } else if (proxyConfig) {
+        return {
+            ...proxyConfig,
+            trustedCAs: proxyConfig.trustedCAs?.map((caDefinition) =>
+                typeof caDefinition !== 'string' && 'cert' in caDefinition
+                    ? { cert: caDefinition.cert.toString('utf8') } // Stringify in case of buffers
+                    : caDefinition
+            ),
+            additionalTrustedCAs: proxyConfig.additionalTrustedCAs?.map((caDefinition) =>
+                'cert' in caDefinition
+                    ? { cert: caDefinition.cert.toString('utf8') } // Stringify in case of buffers
+                    : caDefinition
+            )
+        }
     }
 }
 

test/integration/proxying/upstream-proxying.spec.ts

@@ -287,7 +287,7 @@ nodeOnly(() => {
                 expect(result.message).to.match(/self(-| )signed certificate/); // Dash varies by Node version
             });
 
-            it("should trust the remote proxy's CA if explicitly specified", async () => {
+            it("should trust the remote proxy's CA if explicitly specified by content", async () => {
                 // Remote server sends fixed response on this one URL:
                 await remoteServer.forGet('/test-url').thenReply(200, "Remote server says hi!");
 
@@ -296,7 +296,51 @@ nodeOnly(() => {
                     proxyConfig: {
                         proxyUrl: intermediateProxy.url,
                         trustedCAs: [
-                            (await fs.readFile('./test/fixtures/untrusted-ca.pem')).toString()
+                            { cert: await fs.readFile('./test/fixtures/untrusted-ca.pem') }
+                        ]
+                    }
+                });
+
+                const response = await request.get(remoteServer.urlFor("/test-url"));
+
+                // We get a successful response
+                expect(response).to.equal("Remote server says hi!");
+                // And it went via the intermediate proxy
+                expect((await proxyEndpoint.getSeenRequests()).length).to.equal(1);
+            });
+
+            it("should trust the remote proxy's CA if explicitly specified by file path", async () => {
+                // Remote server sends fixed response on this one URL:
+                await remoteServer.forGet('/test-url').thenReply(200, "Remote server says hi!");
+
+                // Mockttp forwards requests via our intermediate proxy
+                await server.forAnyRequest().thenPassThrough({
+                    proxyConfig: {
+                        proxyUrl: intermediateProxy.url,
+                        trustedCAs: [
+                            { certPath: './test/fixtures/untrusted-ca.pem' }
+                        ]
+                    }
+                });
+
+                const response = await request.get(remoteServer.urlFor("/test-url"));
+
+                // We get a successful response
+                expect(response).to.equal("Remote server says hi!");
+                // And it went via the intermediate proxy
+                expect((await proxyEndpoint.getSeenRequests()).length).to.equal(1);
+            });
+
+            it("should trust the remote proxy's CA if explicitly specified as additional", async () => {
+                // Remote server sends fixed response on this one URL:
+                await remoteServer.forGet('/test-url').thenReply(200, "Remote server says hi!");
+
+                // Mockttp forwards requests via our intermediate proxy
+                await server.forAnyRequest().thenPassThrough({
+                    proxyConfig: {
+                        proxyUrl: intermediateProxy.url,
+                        additionalTrustedCAs: [
+                            { cert: (await fs.readFile('./test/fixtures/untrusted-ca.pem')).toString() }
                         ]
                     }
                 });