cells: use core domain endpoint only if hostname resolable

Motivation:

Recently we have observe cases whee tunnel connections failed with:

java.lang.NullPointerException: null
        at java.base/java.net.Socket.<init>(Socket.java:501)
        at java.base/java.net.Socket.<init>(Socket.java:319)
        at java.base/javax.net.DefaultSocketFactory.createSocket(SocketFactory.java:277)
        at dmg.cells.network.LocationManagerConnector.connect(LocationManagerConnector.java:66)
        at dmg.cells.network.LocationManagerConnector.run(LocationManagerConnector.java:96)
        at dmg.cells.nucleus.CellNucleus.lambda$wrapLoggingContext$2(CellNucleus.java:725)
        at java.base/java.lang.Thread.run(Thread.java:840)

This is possible only if hostname can't be resolve. Such tunnels die and
never re-connect.

Modification:
Update zookeeper node update logic to ensure that we accept endpoint
only if hostname is resolvable.

Result:
More robust tunnel handling.

Issue: #5326
Acked-by: Marina Sahakyan
Target: master, 10.2
Require-book: no
Require-notes: yes

Author: kofemann

modules/cells/src/main/java/dmg/cells/services/LocationManager.java

@@ -58,6 +58,7 @@
 import org.apache.curator.framework.recipes.cache.ChildData;
 import org.apache.curator.framework.recipes.cache.PathChildrenCache;
 import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent.Type;
 import org.apache.curator.utils.CloseableUtils;
 import org.apache.curator.utils.ZKPaths;
 import org.dcache.ssl.CanlSslSocketCreator;
@@ -176,7 +177,9 @@ public CoreDomainInfo(byte[] bytes) {
                     switch (entry.getScheme()) {
                         case "tls":
                         case "tcp":
-                            endpoints.add(entry);
+                            if (canResoveHost(entry)) {
+                                endpoints.add(entry);
+                            }
                             break;
                         default:
                             LOGGER.warn("Unknown Scheme for LocationManager Cores: {}", entry);
@@ -189,6 +192,26 @@ public CoreDomainInfo(byte[] bytes) {
             }
         }
 
+        /**
+         * Check if hostname provided by urc can be resolved.
+         * @param endpoint core domain endpoint
+         * @return true it host name can be resolved. Otherwise false.
+         */
+        private boolean canResoveHost(URI endpoint) {
+            var h = endpoint.getHost();
+            if (h == null) {
+                LOGGER.warn("Ignoring URL without host: {}", endpoint);
+                return false;
+            }
+            try {
+                var ip = java.net.InetAddress.getByName(h);
+            } catch (UnknownHostException e) {
+                LOGGER.warn("Ignoring unknown host: {} : {}", endpoint, e.toString());
+                return false;
+            }
+            return true;
+        }
+
         void addCore(String scheme, String host, int port) {
             switch (scheme) {
                 case "tls":
@@ -385,46 +408,38 @@ public void update(PathChildrenCacheEvent event) {
             LOGGER.info("{}", event);
             String cell;
 
+            if (hasNoData(event.getData())) {
+                LOGGER.warn("Ignoring empty event {}", event.getType());
+                return;
+            }
+
+            String domain = ZKPaths.getNodeFromPath(event.getData().getPath());
+
             switch (event.getType()) {
                 case CHILD_REMOVED:
-                    cell = connectors.remove(ZKPaths.getNodeFromPath(event.getData().getPath()));
+                    cell = connectors.remove(domain);
                     if (cell != null) {
                         killConnector(cell);
                     }
                     break;
                 case CHILD_UPDATED:
-                    if (hasNoData(event.getData())) {
-                        LOGGER.warn("Ignoring empty data on UPDATED for {}", event.getData().getPath());
-                        break;
-                    }
-                    cell = connectors.remove(
-                          ZKPaths.getNodeFromPath(event.getData().getPath()));
-                    if (cell != null) {
-                        killConnector(cell);
-                    }
-                    // fall through
                 case CHILD_ADDED:
-                    if (hasNoData(event.getData())) {
-                        LOGGER.warn("Ignoring empty data on ADDED for {}", event.getData().getPath());
+
+                    CoreDomainInfo info = infoFromZKEvent(event);
+                    if (info.endpoints.isEmpty()) {
+                        LOGGER.warn("Ignoring invalid core URI", domain);
                         break;
                     }
 
-                    //Log if the Core Domain Information received is incompatible with previous
-                    CoreDomainInfo info = infoFromZKEvent(event);
-                    String domain = ZKPaths.getNodeFromPath(event.getData().getPath());
+                    if (event.getType() == Type.CHILD_UPDATED) {
+                        cell = connectors.remove(domain);
+                        if (cell != null) {
+                            killConnector(cell);
+                        }
+                    }
 
                     try {
                         if (shouldConnectTo(domain)) {
-                            cell = connectors.remove(domain);
-                            if (cell != null) {
-                                LOGGER.error(
-                                      "About to create tunnel to core domain {}, but to my surprise "
-                                            +
-                                            "a tunnel called {} already exists. Will kill it. Please contact "
-                                            +
-                                            "[email protected].", domain, cell);
-                                killConnector(cell);
-                            }
                             cell = connectors.put(domain, startConnector(domain, info));
                             if (cell != null) {
                                 LOGGER.error(