Add support for no-proxy passthrough configurations

Author: pimterry

src/rules/requests/request-handlers.ts

@@ -29,7 +29,8 @@ import {
     h2HeadersToH1,
     isAbsoluteUrl,
     cleanUpHeaders,
-    isMockttpBody
+    isMockttpBody,
+    matchesNoProxy
 } from '../../util/request-utils';
 import { streamToBuffer, asBuffer } from '../../util/buffer-utils';
 import { isLocalPortActive, isSocketLoop } from '../../util/socket-util';
@@ -423,6 +424,26 @@ export interface ProxyConfig {
      * For example: http://..., socks5://..., pac+http://...
      */
     proxyUrl: string;
+
+    /**
+     * A list of no-proxy values, matching URLs which should not be proxied.
+     *
+     * This is a common proxy feature, but unfortunately isn't standardized. See
+     * https://about.gitlab.com/blog/2021/01/27/we-need-to-talk-no-proxy/ for some
+     * background. This implementation is intended to match Curl's behaviour, and
+     * any differences are a bug.
+     *
+     * The currently supported formats are:
+     * - example.com (matches domain and all subdomains)
+     * - example.com:443 (matches domain and all subdomains, but only on that port)
+     * - 10.0.0.1 (matches IP, but only when used directly - does not resolve domains)
+     *
+     * Some other formats (e.g. leading dots or *.) will work, but the leading
+     * characters are ignored. More formats may be added in future, e.g. CIDR ranges.
+     * To maximize compatibility with values used elsewhere, unrecognized formats
+     * will generally be ignored, but may match in unexpected ways.
+     */
+    noProxy?: string[];
 }
 
 export interface PassThroughLookupOptions {
@@ -480,7 +501,7 @@ export interface PassThroughHandlerOptions {
     /**
      * Upstream proxy configuration: pass through requests via this proxy
      */
-    proxyConfig?: ProxyConfig
+    proxyConfig?: ProxyConfig;
 
     /**
      * Custom DNS options, to allow configuration of the resolver used
@@ -872,32 +893,38 @@ const KeepAliveAgents = isNode
         })
     } : {};
 
-function getAgent(options: {
+function getAgent({
+    protocol, hostname, port, tryHttp2, keepAlive, proxyConfig
+}: {
     protocol: 'http:' | 'https:' | undefined,
+    hostname: string,
+    port: number,
     tryHttp2: boolean,
     keepAlive: boolean
     proxyConfig: ProxyConfig | undefined,
 }): {} | undefined {
-    if (options.proxyConfig && options.proxyConfig.proxyUrl) {
+    if (proxyConfig && proxyConfig.proxyUrl) {
         // If there's a (non-empty) proxy configured, use it. We require non-empty because empty strings
         // will fall back to detecting from the environment, which is likely to behave unexpectedly.
 
-        // We notably ignore HTTP/2 upstream in this case: it's complicated to mix that up with proxying
-        // so for now we ignore it entirely.
-        return new ProxyAgent(options.proxyConfig.proxyUrl);
-    } else {
-        if (options.tryHttp2 && options.protocol === 'https:') {
-            // H2 wrapper takes multiple agents, uses the appropriate one for the detected protocol.
-            // We notably never use H2 upstream for plaintext, it's rare and we can't use ALPN to detect it.
-            return { https: KeepAliveAgents['https:'], http2: undefined };
-        } else if (options.keepAlive) {
-            // HTTP/1.1 or HTTP/1 with explicit keep-alive
-            return KeepAliveAgents[options.protocol || 'http:']
-        } else {
-            // HTTP/1 without KA - just send the request with no agent
-            return undefined;
+        if (!matchesNoProxy(hostname, port, proxyConfig.noProxy)) {
+            // We notably ignore HTTP/2 upstream in this case: it's complicated to mix that up with proxying
+            // so for now we ignore it entirely.
+            return new ProxyAgent(proxyConfig.proxyUrl);
         }
     }
+
+    if (tryHttp2 && protocol === 'https:') {
+        // H2 wrapper takes multiple agents, uses the appropriate one for the detected protocol.
+        // We notably never use H2 upstream for plaintext, it's rare and we can't use ALPN to detect it.
+        return { https: KeepAliveAgents['https:'], http2: undefined };
+    } else if (keepAlive) {
+        // HTTP/1.1 or HTTP/1 with explicit keep-alive
+        return KeepAliveAgents[protocol || 'http:']
+    } else {
+        // HTTP/1 without KA - just send the request with no agent
+        return undefined;
+    }
 }
 
 export class PassThroughHandler extends Serializable implements RequestHandler {
@@ -1216,22 +1243,25 @@ export class PassThroughHandler extends Serializable implements RequestHandler {
                 : http.request
         ) as typeof https.request;
 
+        const effectivePort = !!port
+            ? parseInt(port, 10)
+            : (protocol === 'https:' ? 443 : 80);
+
         let family: undefined | 4 | 6;
         if (hostname === 'localhost') {
             // Annoying special case: some localhost servers listen only on either ipv4 or ipv6.
             // Very specific situation, but a very common one for development use.
             // We need to work out which one family is, as Node sometimes makes bad choices.
-            const portToTest = !!port
-                ? parseInt(port, 10)
-                : (protocol === 'https:' ? 443 : 80);
 
-            if (await isLocalPortActive('::1', portToTest)) family = 6;
+            if (await isLocalPortActive('::1', effectivePort)) family = 6;
             else family = 4;
         }
 
         // Mirror the keep-alive-ness of the incoming request in our outgoing request
         const agent = getAgent({
             protocol: (protocol || undefined) as 'http:' | 'https:' | undefined,
+            hostname: hostname!,
+            port: effectivePort,
             tryHttp2: shouldTryH2Upstream,
             keepAlive: shouldKeepAlive(clientReq),
             proxyConfig: this.proxyConfig

src/util/request-utils.ts

@@ -65,6 +65,36 @@ export const shouldKeepAlive = (req: OngoingRequest): boolean =>
     req.headers['connection'] !== 'close' &&
     req.headers['proxy-connection'] !== 'close';
 
+export const matchesNoProxy = (hostname: string, portNum: number, noProxyValues: string[] | undefined) => {
+    if (!noProxyValues || noProxyValues.length === 0) return false; // Skip everything in the common case.
+
+    const port = portNum.toString();
+    const hostParts = hostname.split('.').reverse();
+
+    return noProxyValues.some((noProxy) => {
+        const [noProxyHost, noProxyPort] = noProxy.split(':') as [string, string | undefined];
+
+        let noProxyParts = noProxyHost.split('.').reverse();
+        const lastPart = noProxyParts[noProxyParts.length - 1];
+        if (lastPart === '' || lastPart === '*') {
+            noProxyParts = noProxyParts.slice(0, -1);
+        }
+
+        if (noProxyPort && port !== noProxyPort) return false;
+
+        for (let i = 0; i < noProxyParts.length; i++) {
+            let noProxyPart = noProxyParts[i];
+            let hostPart = hostParts[i];
+
+            if (hostPart === undefined) return false; // No-proxy is longer than hostname
+            if (noProxyPart !== hostPart) return false; // Mismatch
+        }
+
+        // If we run out of no-proxy parts with no mismatch then we've matched
+        return true;
+    });
+}
+
 export const setHeaders = (response: http.ServerResponse, headers: Headers) => {
     Object.keys(headers).forEach((header) => {
         let value = headers[header];

test/integration/proxy.spec.ts

@@ -1545,7 +1545,7 @@ nodeOnly(() => {
         describe("when configured to transform requests automatically", () => {
 
             beforeEach(async () => {
-                server = getLocal({ debug: true });
+                server = getLocal();
                 await server.start();
                 process.env = _.merge({}, process.env, server.proxyEnv);
 
@@ -1868,7 +1868,7 @@ nodeOnly(() => {
         describe("when configured to transform responses automatically", () => {
 
             beforeEach(async () => {
-                server = getLocal({ debug: true });
+                server = getLocal();
                 await server.start();
                 process.env = _.merge({}, process.env, server.proxyEnv);
 
@@ -2166,11 +2166,11 @@ nodeOnly(() => {
 
         describe("when configured to use an upstream proxy", () => {
 
-            const intermediateProxy = getLocal({ debug: true });
+            const intermediateProxy = getLocal();
             let proxyEndpoint: MockedEndpoint;
 
             beforeEach(async () => {
-                server = getLocal({ debug: true });
+                server = getLocal();
                 await server.start();
 
                 await intermediateProxy.start();
@@ -2200,6 +2200,143 @@ nodeOnly(() => {
                 // And it went via the intermediate proxy
                 expect((await proxyEndpoint.getSeenRequests()).length).to.equal(1);
             });
+
+            it("should skip the proxy if the target is in the no-proxy list", async () => {
+                // Remote server sends fixed response on this one URL:
+                await remoteServer.get('/test-url').thenReply(200, "Remote server says hi!");
+
+                // Mockttp forwards requests via our intermediate proxy
+                await server.anyRequest().thenPassThrough({
+                    proxyConfig: {
+                        proxyUrl: intermediateProxy.url,
+                        noProxy: ['localhost']
+                    }
+                });
+
+                const response = await request.get(remoteServer.urlFor("/test-url"));
+
+                // We get a successful response
+                expect(response).to.equal("Remote server says hi!");
+
+                // And it didn't use the proxy
+                expect((await proxyEndpoint.getSeenRequests()).length).to.equal(0);
+            });
+
+            it("should skip the proxy if the target is in the no-proxy list with a matching port", async () => {
+                // Remote server sends fixed response on this one URL:
+                await remoteServer.get('/test-url').thenReply(200, "Remote server says hi!");
+
+                // Mockttp forwards requests via our intermediate proxy
+                await server.anyRequest().thenPassThrough({
+                    proxyConfig: {
+                        proxyUrl: intermediateProxy.url,
+                        noProxy: [`localhost:${remoteServer.port}`]
+                    }
+                });
+
+                const response = await request.get(remoteServer.urlFor("/test-url"));
+
+                // We get a successful response
+                expect(response).to.equal("Remote server says hi!");
+
+                // And it didn't use the proxy
+                expect((await proxyEndpoint.getSeenRequests()).length).to.equal(0);
+            });
+
+            it("should skip the proxy if the target's implicit port is in the no-proxy list", async () => {
+                // Mockttp forwards requests via our intermediate proxy
+                await server.anyRequest().thenPassThrough({
+                    proxyConfig: {
+                        proxyUrl: intermediateProxy.url,
+                        noProxy: ['example.com:80']
+                    }
+                });
+
+                await request.get('http://example.com/').catch(() => {});
+
+                // And it didn't use the proxy
+                expect((await proxyEndpoint.getSeenRequests()).length).to.equal(0);
+            });
+
+            it("should skip the proxy if a suffix of the target is in the no-proxy list", async () => {
+                // Remote server sends fixed response on this one URL:
+                await remoteServer.get('/test-url').thenReply(200, "Remote server says hi!");
+
+                // Mockttp forwards requests via our intermediate proxy
+                await server.anyRequest().thenPassThrough({
+                    proxyConfig: {
+                        proxyUrl: intermediateProxy.url,
+                        noProxy: ['localhost']
+                    }
+                });
+
+                const response = await request.get(
+                    `http://test-subdomain.localhost:${remoteServer.port}/test-url`
+                );
+
+                // We get a successful response
+                expect(response).to.equal("Remote server says hi!");
+
+                // And it didn't use the proxy
+                expect((await proxyEndpoint.getSeenRequests()).length).to.equal(0);
+            });
+
+            it("should not skip the proxy if an unrelated URL is in the no-proxy list", async () => {
+                // Remote server sends fixed response on this one URL:
+                await remoteServer.get('/test-url').thenReply(200, "Remote server says hi!");
+
+                // Mockttp forwards requests via our intermediate proxy
+                await server.anyRequest().thenPassThrough({
+                    proxyConfig: {
+                        proxyUrl: intermediateProxy.url,
+                        noProxy: ['example.com']
+                    }
+                });
+
+                const response = await request.get(remoteServer.urlFor("/test-url"));
+
+                // We get a successful response
+                expect(response).to.equal("Remote server says hi!");
+
+                // And it went via the intermediate proxy
+                expect((await proxyEndpoint.getSeenRequests()).length).to.equal(1);
+            });
+
+            it("should not skip the proxy if the target's port is not in the no-proxy list", async () => {
+                // Remote server sends fixed response on this one URL:
+                await remoteServer.get('/test-url').thenReply(200, "Remote server says hi!");
+
+                // Mockttp forwards requests via our intermediate proxy
+                await server.anyRequest().thenPassThrough({
+                    proxyConfig: {
+                        proxyUrl: intermediateProxy.url,
+                        noProxy: ['localhost:1234']
+                    }
+                });
+
+                const response = await request.get(remoteServer.urlFor("/test-url"));
+
+                // We get a successful response
+                expect(response).to.equal("Remote server says hi!");
+
+                // And it went via the intermediate proxy
+                expect((await proxyEndpoint.getSeenRequests()).length).to.equal(1);
+            });
+
+            it("should not skip the proxy if the target's implicit port is not in the no-proxy list", async () => {
+                // Mockttp forwards requests via our intermediate proxy
+                await server.anyRequest().thenPassThrough({
+                    proxyConfig: {
+                        proxyUrl: intermediateProxy.url,
+                        noProxy: ['example.com:443']
+                    }
+                });
+
+                await request.get('http://example.com/').catch(() => {});
+
+                // And it didn't use the proxy
+                expect((await proxyEndpoint.getSeenRequests()).length).to.equal(1);
+            });
         });
 
         describe("when configured with custom DNS options", function () {