Merge pull request ceph#62011 from kchheda3/wip-allow-multiple-rgws

rgw/frontend: Allow multiple RGWs to run on same port on same host

Reviewed-by: Adam C. Emerson <[email protected]>

Author: cbodley

PendingReleaseNotes

@@ -21,6 +21,8 @@
 * RGW: Adding missing quotes to the ETag values returned by S3 CopyPart,
   PostObject and CompleteMultipartUpload responses.
 * RGW: Added support for S3 GetObjectAttributes.
+* RGW: Added BEAST frontend option 'so_reuseport' which facilitates running multiple
+        RGW instances on the same host by sharing a single TCP port.
 
 * RBD: All Python APIs that produce timestamps now return "aware" `datetime`
   objects instead of "naive" ones (i.e. those including time zone information

doc/radosgw/frontends.rst

@@ -135,6 +135,17 @@ Options
 :Default: ``16384``
 :Maximum: ``65536``
 
+``so_reuseport``
+
+:Description:  If set allows multiple RGW instances on a host to listen on the same TCP port.
+
+              ``1`` Enable running multiple RGW on same port.
+
+              ``0`` Disallow running multiple RGW on same port.
+
+:Type: Integer (0 or 1)
+:Default: 0
+
 
 Generic Options
 ===============

src/rgw/rgw_asio_frontend.cc

@@ -707,8 +707,12 @@ int AsioFrontend::init()
       l.use_nodelay = (nodelay->second == "1");
     }
   }
-  
 
+  bool reuse_port = false;
+  auto reuse_port_it = config.find("so_reuseport");
+  if (reuse_port_it != config.end()) {
+    reuse_port = (reuse_port_it->second == "1");
+  }
   bool socket_bound = false;
   // start listeners
   for (auto& l : listeners) {
@@ -733,7 +737,21 @@ int AsioFrontend::init()
       }
     }
 
-    l.acceptor.set_option(tcp::acceptor::reuse_address(true));
+    if (reuse_port) {
+      // setting option |SO_REUSEPORT| allows running of multiple rgw processes on
+      // the same port. Can read more about the implementation here.
+      // https://web.git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git/commit/?id=c617f398edd4db2b8567a28e899a88f8f574798d
+      int one = 1;
+      if (setsockopt(l.acceptor.native_handle(), SOL_SOCKET,
+                     SO_REUSEADDR | SO_REUSEPORT, &one, sizeof(one)) == -1) {
+        lderr(ctx()) << "setsockopt SO_REUSEADDR | SO_REUSEPORT failed:" <<
+ dendl;
+        return -1;
+      }
+    } else {
+      l.acceptor.set_option(tcp::acceptor::reuse_address(true));
+    }
+
     l.acceptor.bind(l.endpoint, ec);
     if (ec) {
       lderr(ctx()) << "failed to bind address " << l.endpoint