HADOOP-19695. Add dual-stack/IPv6 Support to HttpServer2

To enable dual-stack or IPv6 support, use InetAddress.getAllByName(hostname) to resolve the IP addresses of a host.
When the system property java.net.preferIPv4Stack is set to true, only IPv4 addresses are returned, and any IPv6 addresses are ignored, so no extra check is needed to exclude IPv6.
When java.net.preferIPv4Stack is false, both IPv4 and IPv6 addresses may be returned, and any IPv6 addresses will also be added as connectors.
To disable IPv4, you need to configure the OS at the system level.

Author: ferdelyi

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java

@@ -23,6 +23,7 @@
 import java.io.InterruptedIOException;
 import java.io.PrintStream;
 import java.net.BindException;
+import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.MalformedURLException;
 import java.net.URI;
@@ -549,25 +550,50 @@ public HttpServer2 build() throws IOException {
       }
 
       for (URI ep : endpoints) {
-        final ServerConnector connector;
+        //
+        // To enable dual-stack or IPv6 support, use InetAddress
+        // .getAllByName(hostname) to resolve the IP addresses of a host.
+        // When the system property java.net.preferIPv4Stack is set to true,
+        // only IPv4 addresses are returned, and any IPv6 addresses are
+        // ignored, so no extra check is needed to exclude IPv6.
+        // When java.net.preferIPv4Stack is false, both IPv4 and IPv6
+        // addresses may be returned, and any IPv6 addresses will also be
+        // added as connectors.
+        // To disable IPv4, you need to configure the OS at the system level.
+        //
+        InetAddress[] addresses = InetAddress.getAllByName(ep.getHost());
+        server = addConnectors(
+            ep, addresses, server, httpConfig, backlogSize, idleTimeout);
+      }
+      server.loadListeners();
+      return server;
+    }
+
+    @VisibleForTesting
+    HttpServer2 addConnectors(
+        URI ep, InetAddress[] addresses, HttpServer2 server,
+        HttpConfiguration httpConfig, int backlogSize, int idleTimeout){
+      for (InetAddress addr : addresses) {
+        ServerConnector connector;
         String scheme = ep.getScheme();
         if (HTTP_SCHEME.equals(scheme)) {
-          connector = createHttpChannelConnector(server.webServer,
-              httpConfig);
+          connector = createHttpChannelConnector(
+              server.webServer, httpConfig);
         } else if (HTTPS_SCHEME.equals(scheme)) {
-          connector = createHttpsChannelConnector(server.webServer,
-              httpConfig);
+          connector = createHttpsChannelConnector(
+              server.webServer, httpConfig);
         } else {
           throw new HadoopIllegalArgumentException(
               "unknown scheme for endpoint:" + ep);
         }
-        connector.setHost(ep.getHost());
+        LOG.debug("Adding connector to WebServer for address {}",
+            addr.getHostAddress());
+        connector.setHost(addr.getHostAddress());
         connector.setPort(ep.getPort() == -1 ? 0 : ep.getPort());
         connector.setAcceptQueueSize(backlogSize);
         connector.setIdleTimeout(idleTimeout);
         server.addListener(connector);
       }
-      server.loadListeners();
       return server;
     }
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java

@@ -30,6 +30,7 @@
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AccessControlList;
 
+import org.eclipse.jetty.server.HttpConfiguration;
 import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.server.handler.StatisticsHandler;
 import org.eclipse.jetty.util.ajax.JSON;
@@ -56,6 +57,7 @@
 import java.io.PrintWriter;
 import java.lang.reflect.Field;
 import java.net.HttpURLConnection;
+import java.net.InetAddress;
 import java.net.URI;
 import java.net.URL;
 import java.util.Arrays;
@@ -635,6 +637,26 @@ public void testRequiresAuthorizationAccess() throws Exception {
     assertFalse(HttpServer2.isInstrumentationAccessAllowed(context, request, response));
   }
 
+  @Test
+  public void testAddConnectors() throws Exception {
+    HttpServer2.Builder builder = new HttpServer2.Builder()
+        .setName("test").setConf(new Configuration()).setFindPort(false);
+    URI endpoint = URI.create("http://testaddress.com:8080/my-app");
+    InetAddress[] addresses = new InetAddress[2];
+    // IPv4 test address
+    addresses[0] = InetAddress.getByName("192.168.1.100");
+    // IPv6 test address
+    addresses[1] = InetAddress.getByName("fd00::1");
+    HttpConfiguration httpConfig = new HttpConfiguration();
+    final int backlogSize = 2048;
+    final int idleTimeout = 1000;
+
+    server = builder.addConnectors(
+        endpoint, addresses, server, httpConfig, backlogSize, idleTimeout);
+    //the expected value is 3: the loopback address and the two addresses
+    assertEquals(server.getListeners().toArray().length, 3);
+  }
+
   @Test public void testBindAddress() throws Exception {
     checkBindAddress("localhost", 0, false).stop();
     // hang onto this one for a bit more testing

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java

@@ -17,6 +17,23 @@
  */
 package org.apache.hadoop.hdfs.server.datanode.web;
 
+import java.io.Closeable;
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.net.BindException;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.net.SocketException;
+import java.net.URI;
+import java.nio.channels.ServerSocketChannel;
+import java.security.GeneralSecurityException;
+import java.util.Enumeration;
+import java.util.Map;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletContext;
+
 import io.netty.bootstrap.ServerBootstrap;
 import io.netty.channel.ChannelFactory;
 import io.netty.channel.ChannelFuture;
@@ -25,14 +42,17 @@
 import io.netty.channel.ChannelOption;
 import io.netty.channel.ChannelPipeline;
 import io.netty.channel.EventLoopGroup;
+import io.netty.channel.WriteBufferWaterMark;
 import io.netty.channel.nio.NioEventLoopGroup;
 import io.netty.channel.socket.SocketChannel;
 import io.netty.channel.socket.nio.NioServerSocketChannel;
-import io.netty.channel.WriteBufferWaterMark;
 import io.netty.handler.codec.http.HttpRequestDecoder;
 import io.netty.handler.codec.http.HttpResponseEncoder;
 import io.netty.handler.ssl.SslHandler;
 import io.netty.handler.stream.ChunkedWriteHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -46,25 +66,6 @@
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.security.ssl.SSLFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.FilterConfig;
-import javax.servlet.ServletContext;
-import java.io.Closeable;
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.net.BindException;
-import java.net.InetSocketAddress;
-import java.net.SocketAddress;
-import java.net.SocketException;
-import java.net.URI;
-import java.nio.channels.ServerSocketChannel;
-import java.security.GeneralSecurityException;
-import java.util.Enumeration;
-import java.util.Map;
 
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ADMIN;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_DEFAULT;
@@ -83,7 +84,7 @@ public class DatanodeHttpServer implements Closeable {
   private static final int HTTP_ACCEPTOR_THREADS = 1;
   // Jetty 9.4.x: Adding one more thread to HTTP_MAX_THREADS.
   private static final int HTTP_MAX_THREADS =
-      HTTP_SELECTOR_THREADS + HTTP_ACCEPTOR_THREADS + 2;
+      HTTP_SELECTOR_THREADS + HTTP_ACCEPTOR_THREADS + 8;
   private final HttpServer2 infoServer;
   private final EventLoopGroup bossGroup;
   private final EventLoopGroup workerGroup;