[C10D] fix slow init due to repeated dns resolution failure (pytorch#159596)

It can be be very slow to repeatedly hit DNS resolution failure, but
its very helpful to have DNS names in logs by default. So we try to use DNS
but if we hit a transient failure we just disable it for the remainder of the
job, logging IP addresses instead.

Fixes pytorch#159007

Pull Request resolved: pytorch#159596
Approved by: https://github.com/d4l3k

Author: wconstab

torch/csrc/distributed/c10d/socket.cpp

@@ -193,38 +193,50 @@ class SocketImpl {
 };
 
 std::string formatSockAddr(const struct ::sockaddr* addr, socklen_t len) {
-  char host[NI_MAXHOST], port[NI_MAXSERV]; // NOLINT
+  // It can be be very slow to repeatedly hit DNS resolution failure, but its
+  // very helpful to have DNS names in logs by default. So we try to use DNS but
+  // if we hit a transient failure we just disable it for the remainder of the
+  // job, logging IP addresses instead. See
+  // https://github.com/pytorch/pytorch/issues/159007
+  static bool disable_getnameinfo = false;
 
-  if (int err = ::getnameinfo(
-          addr, len, host, NI_MAXHOST, port, NI_MAXSERV, NI_NUMERICSERV)) {
-    C10D_WARNING(
-        "The hostname of the client socket cannot be retrieved. err={}", err);
+  char host[NI_MAXHOST], port[NI_MAXSERV]; // NOLINT
 
-    // if we can't resolve the hostname, display the IP address
+  if (!disable_getnameinfo) {
+    int err = ::getnameinfo(
+        addr, len, host, NI_MAXHOST, port, NI_MAXSERV, NI_NUMERICSERV);
+    if (err != 0) {
+      C10D_WARNING(
+          "The hostname of the client socket cannot be retrieved. err={}", err);
+      disable_getnameinfo = true;
+    }
+  }
+  // if getnameinfo failed, disable would be set
+  if (!disable_getnameinfo) {
     if (addr->sa_family == AF_INET) {
-      struct sockaddr_in* psai = (struct sockaddr_in*)&addr;
-      // NOLINTNEXTLINE(*array*)
-      char ip[INET_ADDRSTRLEN];
-      if (inet_ntop(addr->sa_family, &(psai->sin_addr), ip, INET_ADDRSTRLEN) !=
-          nullptr) {
-        return fmt::format("{}:{}", ip, psai->sin_port);
-      }
-    } else if (addr->sa_family == AF_INET6) {
-      struct sockaddr_in6* psai = (struct sockaddr_in6*)&addr;
-      // NOLINTNEXTLINE(*array*)
-      char ip[INET6_ADDRSTRLEN];
-      if (inet_ntop(
-              addr->sa_family, &(psai->sin6_addr), ip, INET6_ADDRSTRLEN) !=
-          nullptr) {
-        return fmt::format("[{}]:{}", ip, psai->sin6_port);
-      }
+      return fmt::format("{}:{}", host, port);
     }
-    return "?UNKNOWN?";
+    return fmt::format("[{}]:{}", host, port);
   }
+  // if we can't resolve the hostname, display the IP address
   if (addr->sa_family == AF_INET) {
-    return fmt::format("{}:{}", host, port);
+    struct sockaddr_in* psai = (struct sockaddr_in*)&addr;
+    // NOLINTNEXTLINE(*array*)
+    char ip[INET_ADDRSTRLEN];
+    if (inet_ntop(addr->sa_family, &(psai->sin_addr), ip, INET_ADDRSTRLEN) !=
+        nullptr) {
+      return fmt::format("{}:{}", ip, psai->sin_port);
+    }
+  } else if (addr->sa_family == AF_INET6) {
+    struct sockaddr_in6* psai = (struct sockaddr_in6*)&addr;
+    // NOLINTNEXTLINE(*array*)
+    char ip[INET6_ADDRSTRLEN];
+    if (inet_ntop(addr->sa_family, &(psai->sin6_addr), ip, INET6_ADDRSTRLEN) !=
+        nullptr) {
+      return fmt::format("[{}]:{}", ip, psai->sin6_port);
+    }
   }
-  return fmt::format("[{}]:{}", host, port);
+  return "?UNKNOWN?";
 }
 } // namespace c10d::detail