feat(cubejs-server-core): support proxy in http agent transport (#9263)

Author: bsod90

packages/cubejs-server-core/package.json

@@ -40,6 +40,8 @@
     "codesandbox-import-utils": "^2.1.12",
     "cross-spawn": "^7.0.1",
     "fs-extra": "^8.1.0",
+    "http-proxy-agent": "^7.0.2",
+    "https-proxy-agent": "^7.0.6",
     "is-docker": "^2.1.1",
     "joi": "^17.8.3",
     "jsonwebtoken": "^9.0.2",

packages/cubejs-server-core/src/core/agentCollect.ts

@@ -1,6 +1,8 @@
 import { getEnv } from '@cubejs-backend/shared';
 import http from 'http';
 import https from 'https';
+import { HttpsProxyAgent } from 'https-proxy-agent';
+import { HttpProxyAgent } from 'http-proxy-agent';
 import fetch from 'node-fetch';
 import crypto from 'crypto';
 import WebSocket from 'ws';
@@ -50,12 +52,12 @@ class WebSocketTransport implements AgentTransport {
       clearTimeout(this.pingTimeout);
       this.onClose();
     });
-  
+
     this.wsClient.on('error', e => {
       connectionPromiseReject(e);
       this.logger('Agent Error', { error: (e.stack || e).toString() });
     });
-  
+
     this.wsClient.on('message', (data: WebSocket.Data) => {
       try {
         const { method, params } = JSON.parse(data.toString());
@@ -103,17 +105,45 @@ class WebSocketTransport implements AgentTransport {
   }
 }
 
+function isOnNoProxyList(url: string): boolean {
+  const noProxy = process.env.NO_PROXY || process.env.no_proxy;
+  if (!noProxy) {
+    return false;
+  }
+
+  const parsedUrl = new URL(url);
+  const { hostname } = parsedUrl;
+  const noProxyList = noProxy.split(',').map((entry) => entry.trim());
+
+  return noProxyList.some((entry) => {
+    if (entry === '*') {
+      return true;
+    }
+    if (entry.startsWith('.')) {
+      return hostname.endsWith(entry);
+    }
+
+    return hostname === entry;
+  });
+}
+
 class HttpTransport implements AgentTransport {
-  private agent: http.Agent | https.Agent;
+  private agent: http.Agent | https.Agent | HttpProxyAgent<string> | HttpsProxyAgent<string>;
 
   public constructor(
     private readonly endpointUrl: string
   ) {
-    const AgentClass = endpointUrl.startsWith('https') ? https.Agent : http.Agent;
-    this.agent = new AgentClass({
+    const agentParams = {
       keepAlive: true,
       maxSockets: getEnv('agentMaxSockets')
-    });
+    };
+    if (!isOnNoProxyList(endpointUrl) && (process.env.http_proxy || process.env.https_proxy)) {
+      this.agent = endpointUrl.startsWith('https') ?
+        new HttpsProxyAgent(process.env.https_proxy, agentParams) :
+        new HttpProxyAgent(process.env.http_proxy, agentParams);
+    } else {
+      this.agent = endpointUrl.startsWith('https') ? new https.Agent(agentParams) : new http.Agent(agentParams);
+    }
   }
 
   public ready() {
@@ -167,7 +197,7 @@ export default async (event: Record<string, any>, endpointUrl: string, logger: a
       const sentAt = new Date().toJSON();
       const result = await transport.send(toFlush.map(r => ({ ...r, sentAt })));
       if (!result && retries > 0) return flush(toFlush, retries - 1);
-  
+
       return true;
     } catch (e: any) {
       if (retries > 0) return flush(toFlush, retries - 1);

yarn.lock

@@ -11335,6 +11335,11 @@ agent-base@^7.0.2, agent-base@^7.1.0:
   dependencies:
     debug "^4.3.4"
 
+agent-base@^7.1.2:
+  version "7.1.3"
+  resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-7.1.3.tgz#29435eb821bc4194633a5b89e5bc4703bafc25a1"
+  integrity sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==
+
 agentkeepalive@^4.1.3:
   version "4.1.4"
   resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-4.1.4.tgz#d928028a4862cb11718e55227872e842a44c945b"
@@ -18204,6 +18209,14 @@ http-proxy-agent@^7.0.0:
     agent-base "^7.1.0"
     debug "^4.3.4"
 
+http-proxy-agent@^7.0.2:
+  version "7.0.2"
+  resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz#9a8b1f246866c028509486585f62b8f2c18c270e"
+  integrity sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==
+  dependencies:
+    agent-base "^7.1.0"
+    debug "^4.3.4"
+
 http-proxy-middleware@^2.0.0, http-proxy-middleware@^2.0.3:
   version "2.0.7"
   resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz#915f236d92ae98ef48278a95dedf17e991936ec6"
@@ -18278,6 +18291,14 @@ https-proxy-agent@^7.0.1:
     agent-base "^7.0.2"
     debug "4"
 
+https-proxy-agent@^7.0.6:
+  version "7.0.6"
+  resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz#da8dfeac7da130b05c2ba4b59c9b6cd66611a6b9"
+  integrity sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==
+  dependencies:
+    agent-base "^7.1.2"
+    debug "4"
+
 human-signals@^1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-1.1.1.tgz#c5b1cd14f50aeae09ab6c59fe63ba3395fe4dfa3"