cephadm/nvmeof: support per-node gateway addresses

Added gateway and discovery address maps to the service specification.
These maps store per-node service addresses. The address is first searched
in the map, then in the spec address configuration. If neither is defined,
the host IP is used as a fallback.

Signed-off-by: Alexander Indenbaum <[email protected]>

Author: Alexander Indenbaum

src/pybind/mgr/cephadm/services/nvmeof.py

@@ -38,6 +38,8 @@ def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonD
         spec = cast(NvmeofServiceSpec, self.mgr.spec_store[daemon_spec.service_name].spec)
         nvmeof_gw_id = daemon_spec.daemon_id
         host_ip = self.mgr.inventory.get_addr(daemon_spec.host)
+        map_addr = spec.addr_map.get(daemon_spec.host) if spec.addr_map else None
+        map_discovery_addr = spec.discovery_addr_map.get(daemon_spec.host) if spec.discovery_addr_map else None
 
         keyring = self.get_keyring_with_caps(self.get_auth_entity(nvmeof_gw_id),
                                              ['mon', 'profile rbd',
@@ -47,8 +49,14 @@ def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonD
         transport_tcp_options = json.dumps(spec.transport_tcp_options) if spec.transport_tcp_options else None
         name = '{}.{}'.format(utils.name_to_config_section('nvmeof'), nvmeof_gw_id)
         rados_id = name[len('client.'):] if name.startswith('client.') else name
-        addr = spec.addr or host_ip
-        discovery_addr = spec.discovery_addr or host_ip
+
+        # The address is first searched in the per node address map,
+        # then in the spec address configuration.
+        # If neither is defined, the host IP is used as a fallback.
+        addr = map_addr or spec.addr or host_ip
+        self.mgr.log.info(f"gateway address: {addr} from {map_addr=} {spec.addr=} {host_ip=}")
+        discovery_addr = map_discovery_addr or spec.discovery_addr or host_ip
+        self.mgr.log.info(f"discovery address: {discovery_addr} from {map_discovery_addr=} {spec.discovery_addr=} {host_ip=}")
         context = {
             'spec': spec,
             'name': name,

src/python-common/ceph/deployment/service_spec.py

@@ -1330,6 +1330,7 @@ def __init__(self,
                  name: Optional[str] = None,
                  group: Optional[str] = None,
                  addr: Optional[str] = None,
+                 addr_map: Optional[Dict[str, str]] = None,
                  port: Optional[int] = None,
                  pool: Optional[str] = None,
                  enable_auth: bool = False,
@@ -1380,6 +1381,7 @@ def __init__(self,
                  {"in_capsule_data_size": 8192, "max_io_qpairs_per_ctrlr": 7},
                  tgt_cmd_extra_args: Optional[str] = None,
                  discovery_addr: Optional[str] = None,
+                 discovery_addr_map: Optional[Dict[str, str]] = None,
                  discovery_port: Optional[int] = None,
                  log_level: Optional[str] = 'INFO',
                  log_files_enabled: Optional[bool] = True,
@@ -1414,6 +1416,8 @@ def __init__(self,
         self.pool = pool
         #: ``addr`` address of the nvmeof gateway
         self.addr = addr
+        #: ``addr_map`` per node address map of the nvmeof gateways
+        self.addr_map = addr_map
         #: ``port`` port of the nvmeof gateway
         self.port = port or 5500
         #: ``name`` name of the nvmeof gateway
@@ -1512,6 +1516,8 @@ def __init__(self,
         self.tgt_cmd_extra_args = tgt_cmd_extra_args
         #: ``discovery_addr`` address of the discovery service
         self.discovery_addr = discovery_addr
+        #: ``discovery_addr_map`` per node address map of the discovery service
+        self.discovery_addr_map = discovery_addr_map
         #: ``discovery_port`` port of the discovery service
         self.discovery_port = discovery_port or 8009
         #: ``log_level`` the nvmeof gateway log level