http: add http.setGlobalProxyFromEnv()

This adds an API to dynamically enable built-in proxy support
for all of fetch() and http.request()/https.request(), so
that users do not have to be aware of them all and configure them
one by one.

Author: joyeecheung

doc/api/http.md

@@ -4361,6 +4361,31 @@ added:
 
 Set the maximum number of idle HTTP parsers.
 
+## `http.setGlobalProxyFromEnv(proxyEnv)`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+* `proxyEnv` {Object} An object containing proxy configuration. This accepts the
+  same options as the `proxyEnv` option accepted by [`Agent`][]
+* Returns: {Function} A function that restores the original agent and dispatcher
+  settings to the state before this `http.setGlobalProxyFromEnv()` is invoked.
+
+Dynamically resets the global configurations to enable built-in proxy support for
+`fetch()` and `http.request()`/`https.request()` at runtime, as an alternative
+to using the `--use-env-proxy` flag or `NODE_USE_ENV_PROXY` environment variable.
+It can also be used to override settings configured from the environment variables.
+
+As this function resets the global configurations, any previously configured
+`http.globalAgent`, `https.globalAgent` or undici global dispatcher would be
+overridden after this function is invoked. It's recommended to invoke it before any
+requests are made and avoid invoking it in the middle of any requests.
+
+See [Built-in Proxy Support][] for details on proxy URL formats and `NO_PROXY`
+syntax.
+
 ## Class: `WebSocket`
 
 <!-- YAML
@@ -4383,6 +4408,9 @@ added:
 When Node.js creates the global agent, if the `NODE_USE_ENV_PROXY` environment variable is
 set to `1` or `--use-env-proxy` is enabled, the global agent will be constructed
 with `proxyEnv: process.env`, enabling proxy support based on the environment variables.
+
+To enable proxy support dynamically and globally, use [`http.setGlobalProxyFromEnv()`][].
+
 Custom agents can also be created with proxy support by passing a
 `proxyEnv` option when constructing the agent. The value can be `process.env`
 if they just want to inherit the configuration from the environment variables,
@@ -4438,6 +4466,65 @@ Or the `--use-env-proxy` flag.
 HTTP_PROXY=http://proxy.example.com:8080 NO_PROXY=localhost,127.0.0.1 node --use-env-proxy client.js
 ```
 
+To enable proxy support dynamically and globally:
+
+```cjs
+const http = require('node:http');
+
+const restore = http.setGlobalProxyFromEnv({
+  http_proxy: 'http://proxy.example.com:8080',
+  https_proxy: 'https://proxy.example.com:8443',
+  no_proxy: 'localhost,127.0.0.1,.internal.example.com',
+});
+
+// Subsequent requests will use the configured proxies
+http.get('http://www.example.com', (res) => {
+  // This request will be proxied through proxy.example.com:8080
+});
+
+fetch('https://www.example.com', (res) => {
+  // This request will be proxied through proxy.example.com:8443
+});
+```
+
+```mjs
+import http from 'node:http';
+
+http.setGlobalProxyFromEnv({
+  http_proxy: 'http://proxy.example.com:8080',
+  https_proxy: 'https://proxy.example.com:8443',
+  no_proxy: 'localhost,127.0.0.1,.internal.example.com',
+});
+
+// Subsequent requests will use the configured proxies
+http.get('http://www.example.com', (res) => {
+  // This request will be proxied through proxy.example.com:8080
+});
+
+fetch('https://www.example.com', (res) => {
+  // This request will be proxied through proxy.example.com:8443
+});
+```
+
+To clear the dynamically enabled global proxy configuration:
+
+```cjs
+const http = require('node:http');
+const restore = http.setGlobalProxyFromEnv({ /* ... */ });
+// Perform requests that will be proxied.
+restore();
+// From now on, requests will no longer be proxied.
+```
+
+```mjs
+import http from 'node:http';
+
+const restore = http.setGlobalProxyFromEnv({ /* ... */ });
+// Perform requests that will be proxied.
+restore();
+// From now on, requests will no longer be proxied.
+```
+
 To create a custom agent with built-in proxy support:
 
 ```cjs
@@ -4501,6 +4588,7 @@ const agent2 = new http.Agent({ proxyEnv: process.env });
 [`http.get()`]: #httpgetoptions-callback
 [`http.globalAgent`]: #httpglobalagent
 [`http.request()`]: #httprequestoptions-callback
+[`http.setGlobalProxyFromEnv()`]: #httpsetglobalproxyfromenvproxyenv
 [`message.headers`]: #messageheaders
 [`message.rawHeaders`]: #messagerawheaders
 [`message.socket`]: #messagesocket

lib/_http_agent.js

@@ -40,7 +40,7 @@ const {
   kProxyConfig,
   checkShouldUseProxy,
   kWaitForProxyTunnel,
-  filterEnvForProxies,
+  getGlobalAgent,
 } = require('internal/http');
 const { AsyncResource } = require('async_hooks');
 const { async_id_symbol } = require('internal/async_hooks').symbols;
@@ -627,9 +627,5 @@ function asyncResetHandle(socket) {
 
 module.exports = {
   Agent,
-  globalAgent: new Agent({
-    keepAlive: true, scheduling: 'lifo', timeout: 5000,
-    // This normalized from both --use-env-proxy and NODE_USE_ENV_PROXY settings.
-    proxyEnv: getOptionValue('--use-env-proxy') ? filterEnvForProxies(process.env) : undefined,
-  }),
+  globalAgent: getGlobalAgent(getOptionValue('--use-env-proxy') ? process.env : undefined, Agent),
 };

lib/http.js

@@ -30,6 +30,7 @@ const httpAgent = require('_http_agent');
 const { ClientRequest } = require('_http_client');
 const { methods, parsers } = require('_http_common');
 const { IncomingMessage } = require('_http_incoming');
+const { ERR_PROXY_INVALID_CONFIG } = require('internal/errors').codes;
 const {
   validateHeaderName,
   validateHeaderValue,
@@ -41,6 +42,11 @@ const {
   Server,
   ServerResponse,
 } = require('_http_server');
+const {
+  parseProxyUrl,
+  getGlobalAgent,
+} = require('internal/http');
+const { URL } = require('internal/url');
 let maxHeaderSize;
 let undici;
 
@@ -123,6 +129,57 @@ function lazyUndici() {
   return undici ??= require('internal/deps/undici/undici');
 }
 
+function setGlobalProxyFromEnv(env) {
+  const httpProxy = parseProxyUrl(env, 'http:');
+  const httpsProxy = parseProxyUrl(env, 'https:');
+  const noProxy = env.no_proxy || env.NO_PROXY;
+
+  if (!httpProxy && !httpsProxy) {
+    return () => {};
+  }
+
+  if (httpProxy && !URL.canParse(httpProxy)) {
+    throw new ERR_PROXY_INVALID_CONFIG(httpProxy);
+  }
+  if (httpsProxy && !URL.canParse(httpsProxy)) {
+    throw new ERR_PROXY_INVALID_CONFIG(httpsProxy);
+  }
+
+  let originalDispatcher, originalHttpsAgent, originalHttpAgent;
+  if (httpProxy || httpsProxy) {
+    // Set it for fetch.
+    const { setGlobalDispatcher, getGlobalDispatcher, EnvHttpProxyAgent } = lazyUndici();
+    const envHttpProxyAgent = new EnvHttpProxyAgent({
+      __proto__: null, httpProxy, httpsProxy, noProxy,
+    });
+    originalDispatcher = getGlobalDispatcher();
+    setGlobalDispatcher(envHttpProxyAgent);
+  }
+
+  if (httpProxy) {
+    originalHttpAgent = module.exports.globalAgent;
+    module.exports.globalAgent = getGlobalAgent(env, httpAgent.Agent);
+  }
+  if (httpsProxy && !!process.versions.openssl) {
+    const https = require('https');
+    originalHttpsAgent = https.globalAgent;
+    https.globalAgent = getGlobalAgent(env, https.Agent);
+  }
+
+  return function restore() {
+    if (originalDispatcher) {
+      const { setGlobalDispatcher } = lazyUndici();
+      setGlobalDispatcher(originalDispatcher);
+    }
+    if (originalHttpAgent) {
+      module.exports.globalAgent = originalHttpAgent;
+    }
+    if (originalHttpsAgent) {
+      require('https').globalAgent = originalHttpsAgent;
+    }
+  };
+}
+
 module.exports = {
   _connectionListener,
   METHODS: methods.toSorted(),
@@ -142,6 +199,7 @@ module.exports = {
     validateInteger(max, 'max', 1);
     parsers.max = max;
   },
+  setGlobalProxyFromEnv,
 };
 
 ObjectDefineProperty(module.exports, 'maxHeaderSize', {

lib/https.js

@@ -50,8 +50,8 @@ const tls = require('tls');
 const {
   kProxyConfig,
   checkShouldUseProxy,
-  filterEnvForProxies,
   kWaitForProxyTunnel,
+  getGlobalAgent,
 } = require('internal/http');
 const { Agent: HttpAgent } = require('_http_agent');
 const {
@@ -602,11 +602,7 @@ Agent.prototype._evictSession = function _evictSession(key) {
   delete this._sessionCache.map[key];
 };
 
-const globalAgent = new Agent({
-  keepAlive: true, scheduling: 'lifo', timeout: 5000,
-  // This normalized from both --use-env-proxy and NODE_USE_ENV_PROXY settings.
-  proxyEnv: getOptionValue('--use-env-proxy') ? filterEnvForProxies(process.env) : undefined,
-});
+const globalAgent = getGlobalAgent(getOptionValue('--use-env-proxy') ? process.env : undefined, Agent);
 
 /**
  * Makes a request to a secure web server.

lib/internal/http.js

@@ -186,11 +186,7 @@ class ProxyConfig {
   }
 }
 
-function parseProxyConfigFromEnv(env, protocol, keepAlive) {
-  // We only support proxying for HTTP and HTTPS requests.
-  if (protocol !== 'http:' && protocol !== 'https:') {
-    return null;
-  }
+function parseProxyUrl(env, protocol) {
   // Get the proxy url - following the most popular convention, lower case takes precedence.
   // See https://about.gitlab.com/blog/we-need-to-talk-no-proxy/#http_proxy-and-https_proxy
   const proxyUrl = (protocol === 'https:') ?
@@ -204,6 +200,20 @@ function parseProxyConfigFromEnv(env, protocol, keepAlive) {
     throw new ERR_PROXY_INVALID_CONFIG(`Invalid proxy URL: ${proxyUrl}`);
   }
 
+  return proxyUrl;
+}
+
+function parseProxyConfigFromEnv(env, protocol, keepAlive) {
+  // We only support proxying for HTTP and HTTPS requests.
+  if (protocol !== 'http:' && protocol !== 'https:') {
+    return null;
+  }
+
+  const proxyUrl = parseProxyUrl(env, protocol);
+  if (proxyUrl === null) {
+    return null;
+  }
+
   // Only http:// and https:// proxies are supported.
   // Ignore instead of throw, in case other protocols are supposed to be
   // handled by the user land.
@@ -244,6 +254,13 @@ function filterEnvForProxies(env) {
   };
 }
 
+function getGlobalAgent(proxyEnv, Agent) {
+  return new Agent({
+    keepAlive: true, scheduling: 'lifo', timeout: 5000,
+    proxyEnv,
+  });
+}
+
 module.exports = {
   kOutHeaders: Symbol('kOutHeaders'),
   kNeedDrain: Symbol('kNeedDrain'),
@@ -257,4 +274,6 @@ module.exports = {
   getNextTraceEventId,
   isTraceHTTPEnabled,
   filterEnvForProxies,
+  getGlobalAgent,
+  parseProxyUrl,
 };

test/client-proxy/test-http-proxy-fetch.mjs

@@ -20,9 +20,7 @@ await once(proxy, 'listening');
 const serverHost = `localhost:${server.address().port}`;
 
 // FIXME(undici:4083): undici currently always tunnels the request over
-// CONNECT if proxyTunnel is not explicitly set to false, but what we
-// need is for it to be automatically false for HTTP requests to be
-// consistent with curl.
+// CONNECT if proxyTunnel is not explicitly set to false.
 const expectedLogs = [{
   method: 'CONNECT',
   url: serverHost,

test/client-proxy/test-http-set-global-proxy-from-env-fetch-empty.mjs

@@ -0,0 +1,21 @@
+// Tests that http.setGlobalProxyFromEnv() is a no-op when no proxy is configured.
+
+import '../common/index.mjs';
+import assert from 'node:assert';
+import { startTestServers, checkProxiedFetch } from '../common/proxy-server.js';
+
+const { proxyLogs, shutdown, httpEndpoint: { requestUrl } } = await startTestServers({
+  httpEndpoint: true,
+});
+
+await checkProxiedFetch({
+  FETCH_URL: requestUrl,
+  SET_GLOBAL_PROXY: JSON.stringify({}),
+}, {
+  stdout: 'Hello world',
+});
+
+shutdown();
+
+// Verify request did NOT go through proxy.
+assert.deepStrictEqual(proxyLogs, []);

test/client-proxy/test-http-set-global-proxy-from-env-fetch-https.mjs

@@ -0,0 +1,38 @@
+// Tests that http.setGlobalProxyFromEnv() works with fetch().
+
+import * as common from '../common/index.mjs';
+import assert from 'node:assert';
+import * as fixtures from '../common/fixtures.mjs';
+import { startTestServers, checkProxiedFetch } from '../common/proxy-server.js';
+
+if (!common.hasCrypto) {
+  common.skip('Needs crypto');
+}
+
+const { proxyLogs, proxyUrl, shutdown, httpsEndpoint: { serverHost, requestUrl } } = await startTestServers({
+  httpsEndpoint: true,
+});
+
+await checkProxiedFetch({
+  FETCH_URL: requestUrl,
+  SET_GLOBAL_PROXY: JSON.stringify({
+    https_proxy: proxyUrl,
+  }),
+  NODE_EXTRA_CA_CERTS: fixtures.path('keys', 'fake-startcom-root-cert.pem'),
+}, {
+  stdout: 'Hello world',
+});
+
+shutdown();
+
+const expectedLogs = [{
+  method: 'CONNECT',
+  url: serverHost,
+  headers: {
+    'connection': 'close',
+    'host': serverHost,
+    'proxy-connection': 'keep-alive',
+  },
+}];
+
+assert.deepStrictEqual(proxyLogs, expectedLogs);

test/client-proxy/test-http-set-global-proxy-from-env-fetch-no-proxy.mjs

@@ -0,0 +1,23 @@
+// Tests that http.setGlobalProxyFromEnv() works with no_proxy configuration.
+
+import '../common/index.mjs';
+import assert from 'node:assert';
+import { startTestServers, checkProxiedFetch } from '../common/proxy-server.js';
+const { proxyLogs, shutdown, proxyUrl, httpEndpoint: { requestUrl } } = await startTestServers({
+  httpEndpoint: true,
+});
+
+await checkProxiedFetch({
+  FETCH_URL: requestUrl,
+  SET_GLOBAL_PROXY: JSON.stringify({
+    http_proxy: proxyUrl,
+    no_proxy: 'localhost',
+  }),
+}, {
+  stdout: 'Hello world',
+});
+
+shutdown();
+
+// Verify request did NOT go through proxy.
+assert.deepStrictEqual(proxyLogs, []);