feat: enable custom lookup for httpclient and fetch

index.d.ts

@@ -2,7 +2,7 @@ import accepts = require('accepts');
 import { AsyncLocalStorage } from 'async_hooks';
 import { EventEmitter } from 'events'
 import { Readable } from 'stream';
-import { Socket } from 'net';
+import { Socket, LookupFunction } from 'net';
 import { IncomingMessage, ServerResponse } from 'http';
 import KoaApplication = require('koa');
 import KoaRouter = require('koa-router');
@@ -317,6 +317,8 @@ declare module 'egg' {
     useHttpClientNext?: boolean;
     /** Allow to use HTTP2 first, only work on `useHttpClientNext = true`. Default is `false` */
     allowH2?: boolean;
+    /** Custom lookup function for DNS resolution */
+    lookup?: LookupFunction;
   }
 
   export interface EggAppConfig {
@@ -1297,4 +1299,4 @@ declare module 'egg' {
   export interface Singleton<T> {
     get(id: string): T;
   }
-}
+}

lib/core/fetch_factory.js

@@ -2,6 +2,8 @@ const debug = require('util').debuglog('egg:lib:core:fetch_factory');
 
 const mainNodejsVersion = parseInt(process.versions.node.split('.')[0]);
 let FetchFactory;
+let fetch;
+let fetchInitialized = false;
 let safeFetch;
 let ssrfFetchFactory;
 
@@ -12,15 +14,31 @@ if (mainNodejsVersion >= 20) {
     FetchFactory = urllib4.FetchFactory;
     debug('urllib4 enable');
 
+
+    fetch = function fetch(url, init) {
+      if (!fetchInitialized) {
+        const clientOptions = {};
+        if (this.config.httpclient?.lookup) {
+          clientOptions.lookup = this.config.httpclient.lookup;
+        }
+        FetchFactory.setClientOptions(clientOptions);
+        fetchInitialized = true;
+      }
+      return FetchFactory.fetch(url, init);
+    };
+
     safeFetch = function safeFetch(url, init) {
       if (!ssrfFetchFactory) {
-        const ssrfConfig = this.config.security.ssrf;
+        const ssrfConfig = this.config.security?.ssrf;
         const clientOptions = {};
         if (ssrfConfig?.checkAddress) {
           clientOptions.checkAddress = ssrfConfig.checkAddress;
         } else {
           this.logger.warn('[egg-security] please configure `config.security.ssrf` first');
         }
+        if (this.config.httpclient?.lookup) {
+          clientOptions.lookup = this.config.httpclient.lookup;
+        }
         ssrfFetchFactory = new FetchFactory();
         ssrfFetchFactory.setClientOptions(clientOptions);
       }
@@ -34,4 +52,5 @@ if (mainNodejsVersion >= 20) {
 module.exports = {
   FetchFactory,
   safeFetch,
+  fetch,
 };

lib/core/httpclient.js

@@ -30,6 +30,7 @@ class HttpClient extends urllib.HttpClient2 {
     } else {
       args.tracer = args.tracer || this.app.tracer;
     }
+    args.lookup = args.lookup ?? this.app.config.httpclient?.lookup;
 
     try {
       return await super.request(url, args);

lib/core/httpclient_next.js

@@ -32,6 +32,7 @@ class HttpClientNext extends HttpClient {
       app,
       defaultArgs: options.request,
       allowH2: options.allowH2,
+      lookup: options.lookup ?? app.config.httpclient?.lookup,
       // use on egg-security ssrf
       // https://github.com/eggjs/egg-security/blob/master/lib/extend/safe_curl.js#L11
       checkAddress: options.checkAddress,
@@ -62,8 +63,12 @@ class HttpClientNext extends HttpClient {
       } else {
         this.app.logger.warn('[egg-security] please configure `config.security.ssrf` first');
       }
+      if (!options.lookup && this.app.config.httpclient.lookup) {
+        options.lookup = this.app.config.httpclient.lookup;
+      }
       this[SSRF_HTTPCLIENT] = new HttpClientNext(this.app, {
         checkAddress: ssrfConfig.checkAddress,
+        lookup: options.lookup,
       });
     }
     return await this[SSRF_HTTPCLIENT].request(url, options);

lib/egg.js

@@ -14,7 +14,7 @@ const Messenger = require('./core/messenger');
 const DNSCacheHttpClient = require('./core/dnscache_httpclient');
 const HttpClient = require('./core/httpclient');
 const HttpClientNext = require('./core/httpclient_next');
-const { FetchFactory, safeFetch } = require('./core/fetch_factory');
+const { FetchFactory, safeFetch, fetch } = require('./core/fetch_factory');
 const createLoggers = require('./core/logger');
 const Singleton = require('./core/singleton');
 const utils = require('./core/utils');
@@ -54,11 +54,9 @@ class EggApplication extends EggCore {
     this.HttpClientNext = HttpClientNext;
     this.FetchFactory = FetchFactory;
     if (FetchFactory) {
-      this.FetchFactory.setClientOptions();
-      this.fetch = FetchFactory.fetch;
+      this.fetch = fetch.bind(this);
       this.safeFetch = safeFetch.bind(this);
     }
-
     this.loader.loadConfig();
 
     /**
@@ -297,11 +295,13 @@ class EggApplication extends EggCore {
    * Create a new HttpClient instance with custom options
    * @param {Object} [options] HttpClient init options
    */
-  createHttpClient(options) {
+  createHttpClient(options = {}) {
     let httpClient;
+    options.lookup = options.lookup ?? this.config.httpclient.lookup;
+
     if (this.config.httpclient.useHttpClientNext || this.config.httpclient.allowH2) {
       httpClient = new this.HttpClientNext(this, options);
-    } else if (this.config.httpclient.enableDNSCache) {
+    } else if (this.config.httpclient?.enableDNSCache) {
       httpClient = new DNSCacheHttpClient(this, options);
     } else {
       httpClient = new this.HttpClient(this, options);
@@ -495,7 +495,7 @@ class EggApplication extends EggCore {
     return this.config.env;
   }
   /* eslint no-empty-function: off */
-  set env(_) {}
+  set env(_) { }
 
   /**
    * app.proxy delegate app.config.proxy
@@ -506,7 +506,7 @@ class EggApplication extends EggCore {
     return this.config.proxy;
   }
   /* eslint no-empty-function: off */
-  set proxy(_) {}
+  set proxy(_) { }
 
   /**
    * create a singleton instance

package.json

@@ -79,10 +79,10 @@
     "eslint": "^8.23.1",
     "eslint-config-egg": "^12.0.0",
     "formstream": "^1.1.1",
-    "https-pem": "^3.0.0",
     "jsdoc": "^3.6.11",
     "koa": "^2.13.4",
     "koa-static": "^5.0.0",
+    "node-forge": "^1.3.3",
     "node-libs-browser": "^2.2.1",
     "pedding": "^1.1.0",
     "prettier": "^2.7.1",

test/fixtures/apps/dns_resolver/app/controller/home.js

@@ -0,0 +1,20 @@
+'use strict';
+
+module.exports = async function () {
+  let args;
+  if (this.query.host) {
+    args = {};
+    args.headers = { host: this.query.host };
+  }
+  if (this.query.Host) {
+    args = {};
+    args.headers = { Host: this.query.Host };
+  }
+  if (this.query.disableDNSCache) {
+    args = { enableDNSCache: false };
+  }
+  const result = await this.curl(this.query.url, args);
+  this.status = result.status;
+  this.set(result.headers);
+  this.body = result.data;
+};

test/fixtures/apps/dns_resolver/app/router.js

@@ -0,0 +1,5 @@
+'use strict';
+
+module.exports = app => {
+  app.get('/', app.controller.home);
+};

test/fixtures/apps/dns_resolver/config/config.default.js

@@ -0,0 +1,33 @@
+'use strict';
+
+exports.httpclient = {
+  lookup: function (hostname, options, callback) {
+    const IP_REGEX = /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/;
+    if (IP_REGEX.test(hostname)) {
+      const family = typeof options.family === 'number' ? options.family : 4;
+      if (options.all) {
+        callback(null, [{ address: hostname, family }]);
+      } else {
+        callback(null, hostname, family);
+      }
+    } else {
+      const resultIp = '127.0.0.1';
+      if (options.all) {
+        callback(null, [{ address: resultIp, family: 4 }]);
+      } else {
+        callback(null, resultIp, 4);
+      }
+    }
+  },
+  request: {
+    timeout: 2000,
+  },
+  httpAgent: {
+    keepAlive: false,
+  },
+  httpsAgent: {
+    keepAlive: false,
+  },
+};
+
+exports.keys = 'test key';

test/fixtures/apps/dns_resolver/package.json

@@ -0,0 +1,3 @@
+{
+  "name": "dns_resolver-app"
+}

test/fixtures/apps/fetch_factory/app/controller/home.js

@@ -0,0 +1,20 @@
+'use strict';
+
+module.exports = async function () {
+  let args;
+  if (this.query.host) {
+    args = {};
+    args.headers = { host: this.query.host };
+  }
+  if (this.query.Host) {
+    args = {};
+    args.headers = { Host: this.query.Host };
+  }
+  if (this.query.disableDNSCache) {
+    args = { enableDNSCache: false };
+  }
+  const result = await this.curl(this.query.url, args);
+  this.status = result.status;
+  this.set(result.headers);
+  this.body = result.data;
+};

test/fixtures/apps/fetch_factory/app/router.js

@@ -0,0 +1,5 @@
+'use strict';
+
+module.exports = app => {
+  app.get('/', app.controller.home);
+};

test/fixtures/apps/fetch_factory/config/config.default.js

@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';

test/fixtures/apps/fetch_factory/package.json

@@ -0,0 +1,3 @@
+{
+  "name": "dns_resolver-app"
+}

test/lib/core/dns_resolver.test.js

@@ -0,0 +1,79 @@
+const utils = require('../../utils');
+const assert = require('assert');
+
+describe('test/lib/core/dns_resolver.test.js', () => {
+  let app;
+  let url1;
+  let url2;
+  let server1;
+  let server2;
+
+  before(async () => {
+    server1 = await utils.startNewLocalServer('127.0.0.1');
+    server2 = await utils.startNewLocalServer('127.0.0.2');
+    if (!server1 || !server2) {
+      throw new Error('start local server failed');
+    }
+    app = utils.app('apps/dns_resolver');
+    await app.ready();
+    url1 = server1.url;
+    url1 = url1.replace('127.0.0.1', 'localhost');
+    url2 = server2.url;
+    url2 = url2.replace('127.0.0.2', 'localhost');
+  });
+
+  after(() => {
+    if (server1?.server?.listening) server1.server.close();
+    if (server2?.server?.listening) server2.server.close();
+  });
+
+  it('should bypass dns resolve', async () => {
+    const res = await app.curl(server1.url + '/get_headers', { dataType: 'json' });
+    assert(res.status === 200);
+  });
+
+  it('should curl work', async () => {
+    const res = await app.curl(url1 + '/get_headers', { dataType: 'json' });
+    assert(res.status === 200);
+  });
+
+  it('should fetch also work', async () => {
+    if (!app.fetch) {
+      return;
+    }
+    const res = await app.fetch(url1 + '/get_headers', { dataType: 'json' });
+    assert(res.status === 200);
+  });
+
+  it('should use dns custom lookup and catch error', async () => {
+    try {
+      if (server1?.server?.listening) await server1.server.close();
+      // will resolve to 127.0.0.1
+      const res = await app.curl(url1 + '/get_headers', { dataType: 'json' });
+      assert(res.status !== 200);
+    } catch (err) {
+      assert(err);
+      assert(err.message.includes('ECONNREFUSED'));
+    }
+  });
+
+  it('should safeCurl also work', async () => {
+    const res = await app.safeCurl(url2 + '/get_headers', { dataType: 'json' });
+    assert(res.status === 200);
+  });
+
+  it('should fetch fail', async () => {
+    if (!app.fetch) {
+      return;
+    }
+    try {
+      if (server1?.server?.listening) await server1.server.close();
+      // will resolve to 127.0.0.1
+      const res = await app.fetch(url1 + '/get_headers', { dataType: 'json' });
+      assert(res.status !== 200);
+    } catch (err) {
+      assert(err);
+      assert(err.message.includes('fetch failed'));
+    }
+  });
+});