feat: enable custom lookup for httpclient and fetch

fix: httpclient_next lookup

Author: Dipper30

lib/core/fetch_factory.js

@@ -21,6 +21,9 @@ if (mainNodejsVersion >= 20) {
         } else {
           this.logger.warn('[egg-security] please configure `config.security.ssrf` first');
         }
+        if (this.config.httpclient.lookup) {
+          clientOptions.lookup = this.config.httpclient.lookup;
+        }
         ssrfFetchFactory = new FetchFactory();
         ssrfFetchFactory.setClientOptions(clientOptions);
       }

lib/core/httpclient.js

@@ -30,6 +30,9 @@ class HttpClient extends urllib.HttpClient2 {
     } else {
       args.tracer = args.tracer || this.app.tracer;
     }
+    if (this.app.config.httpclient?.lookup) {
+      args.lookup = this.app.config.httpclient.lookup;
+    }
 
     try {
       return await super.request(url, args);

lib/core/httpclient_next.js

@@ -32,6 +32,7 @@ class HttpClientNext extends HttpClient {
       app,
       defaultArgs: options.request,
       allowH2: options.allowH2,
+      lookup: options.lookup || app.config.httpclient?.lookup,
       // use on egg-security ssrf
       // https://github.com/eggjs/egg-security/blob/master/lib/extend/safe_curl.js#L11
       checkAddress: options.checkAddress,
@@ -62,8 +63,12 @@ class HttpClientNext extends HttpClient {
       } else {
         this.app.logger.warn('[egg-security] please configure `config.security.ssrf` first');
       }
+      if (!options.options && this.app.config.httpclient.lookup) {
+        options.lookup = this.app.config.httpclient.lookup;
+      }
       this[SSRF_HTTPCLIENT] = new HttpClientNext(this.app, {
         checkAddress: ssrfConfig.checkAddress,
+        lookup: options.lookup,
       });
     }
     return await this[SSRF_HTTPCLIENT].request(url, options);

lib/egg.js

@@ -52,15 +52,15 @@ class EggApplication extends EggCore {
     this.ContextHttpClient = ContextHttpClient;
     this.HttpClient = HttpClient;
     this.HttpClientNext = HttpClientNext;
+    this.loader.loadConfig();
     this.FetchFactory = FetchFactory;
     if (FetchFactory) {
-      this.FetchFactory.setClientOptions();
+      const lookup = this.config?.httpclient?.lookup;
+      this.FetchFactory.setClientOptions({ lookup });
       this.fetch = FetchFactory.fetch;
       this.safeFetch = safeFetch.bind(this);
     }
 
-    this.loader.loadConfig();
-
     /**
      * messenger instance
      * @member {Messenger}
@@ -297,11 +297,15 @@ class EggApplication extends EggCore {
    * Create a new HttpClient instance with custom options
    * @param {Object} [options] HttpClient init options
    */
-  createHttpClient(options) {
+  createHttpClient(options = {}) {
     let httpClient;
+    if (this.config.httpclient?.lookup) {
+      options.lookup = this.config.httpclient.lookup
+    }
+
     if (this.config.httpclient.useHttpClientNext || this.config.httpclient.allowH2) {
       httpClient = new this.HttpClientNext(this, options);
-    } else if (this.config.httpclient.enableDNSCache) {
+    } else if (this.config.httpclient?.enableDNSCache) {
       httpClient = new DNSCacheHttpClient(this, options);
     } else {
       httpClient = new this.HttpClient(this, options);
@@ -495,7 +499,7 @@ class EggApplication extends EggCore {
     return this.config.env;
   }
   /* eslint no-empty-function: off */
-  set env(_) {}
+  set env(_) { }
 
   /**
    * app.proxy delegate app.config.proxy
@@ -506,7 +510,7 @@ class EggApplication extends EggCore {
     return this.config.proxy;
   }
   /* eslint no-empty-function: off */
-  set proxy(_) {}
+  set proxy(_) { }
 
   /**
    * create a singleton instance

test/fixtures/apps/dns_resolver/app/controller/home.js

@@ -0,0 +1,20 @@
+'use strict';
+
+module.exports = function* () {
+  let args;
+  if (this.query.host) {
+    args = {};
+    args.headers = { host: this.query.host };
+  }
+  if (this.query.Host) {
+    args = {};
+    args.headers = { Host: this.query.Host };
+  }
+  if (this.query.disableDNSCache) {
+    args = { enableDNSCache: false };
+  }
+  const result = yield this.curl(this.query.url, args);
+  this.status = result.status;
+  this.set(result.headers);
+  this.body = result.data;
+};

test/fixtures/apps/dns_resolver/app/router.js

@@ -0,0 +1,5 @@
+'use strict';
+
+module.exports = app => {
+  app.get('/', app.controller.home);
+};

test/fixtures/apps/dns_resolver/config/config.default.js

@@ -0,0 +1,40 @@
+'use strict';
+
+exports.httpclient = {
+  lookup: function (hostname, options, callback) {
+    const IP_REGEX = /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/;
+    if (IP_REGEX.test(hostname)) {
+      const family = typeof options.family === 'number' ? options.family : 4;
+      console.log(
+        `[dns-resolver] literal IP ${hostname} lookup, bypassing cache`
+      );
+      if (options.all) {
+        callback(null, [{ address: hostname, family }]);
+      } else {
+        callback(null, hostname, family);
+      }
+    } else {
+      console.log(
+        `[dns-resolver] custom lookup for hostname: ${hostname}`
+      );
+
+      const resultIp = '127.0.0.1'
+      if (options.all) {
+        callback(null, [{ address: resultIp, family: 4 }]);
+      } else {
+        callback(null, resultIp, 4);
+      }
+    }
+  },
+  request: {
+    timeout: 2000,
+  },
+  httpAgent: {
+    keepAlive: false,
+  },
+  httpsAgent: {
+    keepAlive: false,
+  },
+};
+
+exports.keys = 'test key';

test/fixtures/apps/dns_resolver/package.json

@@ -0,0 +1,3 @@
+{
+  "name": "dns_resolver-app"
+}

test/lib/core/dns_resolver.test.js

@@ -0,0 +1,107 @@
+const utils = require('../../utils');
+const Koa = require('koa');
+const http = require('http');
+const assert = require('assert');
+
+const startServer = (ip = '127.0.0.1') => {
+  let localServer;
+  return new Promise((resolve, reject) => {
+    const app = new Koa();
+    app.use(async ctx => {
+      if (ctx.path === '/get_headers') {
+        ctx.body = {
+          headers: ctx.request.headers,
+          host: ctx.request.headers.host,
+        };
+        return;
+      }
+      ctx.body = `${ctx.method} ${ctx.path}`;
+    });
+    localServer = http.createServer(app.callback());
+
+    localServer.listen(0, err => {
+      if (err) return reject(err);
+      console.log(`Local test server ${ip} started on port`, localServer.address().port);
+      return resolve({ url: `http://${ip}:` + localServer.address().port, server: localServer });
+    });
+  });
+};
+
+
+
+describe('test/lib/core/dns_resolver.test.js', () => {
+  let app;
+  let url1;
+  let url2;
+  let server1;
+  let server2;
+
+  before(async () => {
+    server1 = await startServer('127.0.0.1')
+    server2 = await startServer('127.0.0.2')
+    if (!server1 || !server2) {
+      throw new Error('start local server failed');
+    }
+    process.once('exit', () => {
+      if (server1 && server1.server) server1.server.close();
+      if (server2 && server2.server) server2.server.close();
+    })
+    app = utils.app('apps/dns_resolver');
+    await app.ready();
+    url1 = server1.url
+    url1 = url1.replace('127.0.0.1', 'localhost');
+    url2 = server2.url
+    url2 = url2.replace('127.0.0.2', 'localhost');
+  });
+
+  it('should surpass dns resolve', async () => {
+    let res = await app.curl(server1.url + '/get_headers', { dataType: 'json' });
+    assert(res.status === 200);
+  });
+
+  it('should curl work', async () => {
+    let res = await app.curl(url1 + '/get_headers', { dataType: 'json' });
+    assert(res.status === 200);
+  });
+
+  it('should fetch also work', async () => {
+    let res = await app.fetch(url1 + '/get_headers', { dataType: 'json' });
+    assert(res.status === 200);
+  });
+
+  it('should use dns custom lookup and catch error', async () => {
+    try {
+      if (server1?.server) await server1.server.close();
+      // will resolve to 127.0.0.1
+      const res = await app.curl(url1 + '/get_headers', { dataType: 'json' });
+      assert(res.status !== 200);
+    } catch (err) {
+      console.error(`\nRequest Error\n`, err.message);
+      assert(err);
+      assert(err.message.includes('ECONNREFUSED'));
+    }
+  });
+
+  it('should safeCurl also work', async () => {
+    let res = await app.curl(url2 + '/get_headers', { dataType: 'json' });
+    assert(res.status === 200);
+  });
+
+  it('should safeFetch also work', async () => {
+    let res = await app.safeFetch(url2 + '/get_headers', { dataType: 'json' });
+    assert(res.status === 200);
+  });
+
+  it('should fetch fail', async () => {
+    try {
+      if (server1?.server) await server1.server.close();
+      // will resolve to 127.0.0.1
+      const res = await app.fetch(url1 + '/get_headers', { dataType: 'json' });
+      assert(res.status !== 200);
+    } catch (err) {
+      console.error(`\nRequest Error\n`, err.message);
+      assert(err);
+      assert(err.message.includes('fetch failed'));
+    }
+  });
+});