Merge pull request ceph#57712 from rhcs-dashboard/dashboard-nvmf-mtls-conf

mgr/dashboard: use secure_channel for grpc requests

Reviewed-by: Adam King <[email protected]>
Reviewed-by: baum <NOT@FOUND>
Reviewed-by: Ernesto Puerta <[email protected]>
Reviewed-by: Redouane Kachach <[email protected]>

Author: nizamial09

src/cephadm/cephadmlib/daemons/nvmeof.py

@@ -76,12 +76,30 @@ def _get_container_mounts(data_dir: str, log_dir: str) -> Dict[str, str]:
         mounts[log_dir] = '/var/log/ceph:z'
         return mounts
 
+    def _get_tls_cert_key_mounts(
+        self, data_dir: str, files: Dict[str, str]
+    ) -> Dict[str, str]:
+        mounts = dict()
+        for fn in [
+            'server_cert',
+            'server_key',
+            'client_cert',
+            'client_key',
+            'root_ca_cert',
+        ]:
+            if fn in files:
+                mounts[
+                    os.path.join(data_dir, fn)
+                ] = f'/{fn.replace("_", ".")}'
+        return mounts
+
     def customize_container_mounts(
         self, ctx: CephadmContext, mounts: Dict[str, str]
     ) -> None:
         data_dir = self.identity.data_dir(ctx.data_dir)
         log_dir = os.path.join(ctx.log_dir, self.identity.fsid)
         mounts.update(self._get_container_mounts(data_dir, log_dir))
+        mounts.update(self._get_tls_cert_key_mounts(data_dir, self.files))
 
     def customize_container_binds(
         self, ctx: CephadmContext, binds: List[List[str]]

src/pybind/mgr/cephadm/inventory.py

@@ -19,6 +19,7 @@
     IngressSpec,
     RGWSpec,
     IscsiServiceSpec,
+    NvmeofServiceSpec,
 )
 from ceph.utils import str_to_datetime, datetime_to_str, datetime_now
 from orchestrator import OrchestratorError, HostSpec, OrchestratorEvent, service_to_daemon_types
@@ -47,6 +48,26 @@ class HostCacheStatus(enum.Enum):
     devices = 'devices'
 
 
+class OrchSecretNotFound(OrchestratorError):
+    def __init__(
+        self,
+        message: Optional[str] = '',
+        entity: Optional[str] = '',
+        service_name: Optional[str] = '',
+        hostname: Optional[str] = ''
+    ):
+        if not message:
+            message = f'No secret found for entity {entity}'
+            if service_name:
+                message += f' with service name {service_name}'
+            if hostname:
+                message += f' with hostname {hostname}'
+        super().__init__(message)
+        self.entity = entity
+        self.service_name = service_name
+        self.hostname = hostname
+
+
 class Inventory:
     """
     The inventory stores a HostSpec for all hosts persistently.
@@ -390,6 +411,31 @@ def _save_certs_and_keys(self, spec: ServiceSpec) -> None:
                     ingress_spec.ssl_key,
                     service_name=ingress_spec.service_name(),
                     user_made=True)
+        elif spec.service_type == 'nvmeof':
+            nvmeof_spec = cast(NvmeofServiceSpec, spec)
+            for cert_attr in [
+                'server_cert',
+                'client_cert',
+                'root_ca_cert'
+            ]:
+                cert = getattr(nvmeof_spec, cert_attr, None)
+                if cert:
+                    self.mgr.cert_key_store.save_cert(
+                        f'nvmeof_{cert_attr}',
+                        cert,
+                        service_name=nvmeof_spec.service_name(),
+                        user_made=True)
+            for key_attr in [
+                'server_key',
+                'client_key',
+            ]:
+                key = getattr(nvmeof_spec, key_attr, None)
+                if key:
+                    self.mgr.cert_key_store.save_key(
+                        f'nvmeof_{key_attr}',
+                        key,
+                        service_name=nvmeof_spec.service_name(),
+                        user_made=True)
 
     def rm(self, service_name: str) -> bool:
         if service_name not in self._specs:
@@ -428,6 +474,12 @@ def _rm_certs_and_keys(self, spec: ServiceSpec) -> None:
         if spec.service_type == 'ingress':
             self.mgr.cert_key_store.rm_cert('ingress_ssl_cert', service_name=spec.service_name())
             self.mgr.cert_key_store.rm_key('ingress_ssl_key', service_name=spec.service_name())
+        if spec.service_type == 'nvmeof':
+            self.mgr.cert_key_store.rm_cert('nvmeof_server_cert', service_name=spec.service_name())
+            self.mgr.cert_key_store.rm_cert('nvmeof_client_cert', service_name=spec.service_name())
+            self.mgr.cert_key_store.rm_cert('nvmeof_root_ca_cert', service_name=spec.service_name())
+            self.mgr.cert_key_store.rm_key('nvmeof_server_key', service_name=spec.service_name())
+            self.mgr.cert_key_store.rm_key('nvmeof_client_key', service_name=spec.service_name())
 
     def get_created(self, spec: ServiceSpec) -> Optional[datetime.datetime]:
         return self.spec_created.get(spec.service_name())
@@ -1853,6 +1905,9 @@ class CertKeyStore():
         'rgw_frontend_ssl_cert',
         'iscsi_ssl_cert',
         'ingress_ssl_cert',
+        'nvmeof_server_cert',
+        'nvmeof_client_cert',
+        'nvmeof_root_ca_cert',
     ]
 
     host_cert = [
@@ -1872,6 +1927,8 @@ class CertKeyStore():
     service_name_key = [
         'iscsi_ssl_key',
         'ingress_ssl_key',
+        'nvmeof_server_key',
+        'nvmeof_client_key',
     ]
 
     known_certs: Dict[str, Any] = {}
@@ -1888,6 +1945,9 @@ def _init_known_cert_key_dicts(self) -> None:
             'rgw_frontend_ssl_cert': {},  # service-name -> cert
             'iscsi_ssl_cert': {},  # service-name -> cert
             'ingress_ssl_cert': {},  # service-name -> cert
+            'nvmeof_server_cert': {},  # service-name -> cert
+            'nvmeof_client_cert': {},  # service-name -> cert
+            'nvmeof_root_ca_cert': {},  # service-name -> cert
             'agent_endpoint_root_cert': Cert(),  # cert
             'mgmt_gw_root_cert': Cert(),  # cert
             'service_discovery_root_cert': Cert(),  # cert
@@ -1909,6 +1969,8 @@ def _init_known_cert_key_dicts(self) -> None:
             'node_exporter_key': {},  # host -> key
             'iscsi_ssl_key': {},  # service-name -> key
             'ingress_ssl_key': {},  # service-name -> key
+            'nvmeof_server_key': {},  # service-name -> key
+            'nvmeof_client_key': {},  # service-name -> key
         }
 
     def get_cert(self, entity: str, service_name: str = '', host: str = '') -> str:

src/pybind/mgr/cephadm/module.py

@@ -88,6 +88,7 @@
     TunedProfileStore,
     NodeProxyCache,
     CertKeyStore,
+    OrchSecretNotFound,
 )
 from .upgrade import CephadmUpgrade
 from .template import TemplateMgr
@@ -3154,6 +3155,30 @@ def cert_store_cert_ls(self) -> Dict[str, Any]:
     def cert_store_key_ls(self) -> Dict[str, Any]:
         return self.cert_key_store.key_ls()
 
+    @handle_orch_error
+    def cert_store_get_cert(
+        self,
+        entity: str,
+        service_name: Optional[str] = None,
+        hostname: Optional[str] = None
+    ) -> str:
+        cert = self.cert_key_store.get_cert(entity, service_name or '', hostname or '')
+        if not cert:
+            raise OrchSecretNotFound(entity=entity, service_name=service_name, hostname=hostname)
+        return cert
+
+    @handle_orch_error
+    def cert_store_get_key(
+        self,
+        entity: str,
+        service_name: Optional[str] = None,
+        hostname: Optional[str] = None
+    ) -> str:
+        key = self.cert_key_store.get_key(entity, service_name or '', hostname or '')
+        if not key:
+            raise OrchSecretNotFound(entity=entity, service_name=service_name, hostname=hostname)
+        return key
+
     @handle_orch_error
     def apply_mon(self, spec: ServiceSpec) -> str:
         return self._apply(spec)

src/pybind/mgr/cephadm/services/nvmeof.py

@@ -53,6 +53,28 @@ def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonD
 
         daemon_spec.keyring = keyring
         daemon_spec.extra_files = {'ceph-nvmeof.conf': gw_conf}
+
+        if spec.enable_auth:
+            if (
+                not spec.client_cert
+                or not spec.client_key
+                or not spec.server_cert
+                or not spec.server_key
+                or not spec.root_ca_cert
+            ):
+                err_msg = 'enable_auth is true but '
+                for cert_key_attr in ['server_key', 'server_cert', 'client_key', 'client_cert', 'root_ca_cert']:
+                    if not hasattr(spec, cert_key_attr):
+                        err_msg += f'{cert_key_attr}, '
+                err_msg += 'attribute(s) missing from nvmeof spec'
+                self.mgr.log.error(err_msg)
+            else:
+                daemon_spec.extra_files['server_cert'] = spec.server_cert
+                daemon_spec.extra_files['client_cert'] = spec.client_cert
+                daemon_spec.extra_files['server_key'] = spec.server_key
+                daemon_spec.extra_files['client_key'] = spec.client_key
+                daemon_spec.extra_files['root_ca_cert'] = spec.root_ca_cert
+
         daemon_spec.final_config, daemon_spec.deps = self.generate_config(daemon_spec)
         daemon_spec.deps = []
         return daemon_spec
@@ -67,9 +89,10 @@ def get_set_cmd_dicts(out: str) -> List[dict]:
 
             for dd in daemon_descrs:
                 assert dd.hostname is not None
+                service_name = dd.service_name()
 
                 if not spec:
-                    logger.warning(f'No ServiceSpec found for {dd.service_name()}')
+                    logger.warning(f'No ServiceSpec found for {service_name}')
                     continue
 
                 ip = utils.resolve_ip(self.mgr.inventory.get_addr(dd.hostname))
@@ -82,7 +105,7 @@ def get_set_cmd_dicts(out: str) -> List[dict]:
                     cmd_dicts.append({
                         'prefix': 'dashboard nvmeof-gateway-add',
                         'inbuf': service_url,
-                        'name': dd.hostname
+                        'name': service_name
                     })
             return cmd_dicts
 
@@ -118,11 +141,12 @@ def post_remove(self, daemon: DaemonDescription, is_failed_deploy: bool) -> None
         """
         # to clean the keyring up
         super().post_remove(daemon, is_failed_deploy=is_failed_deploy)
+        service_name = daemon.service_name()
 
         # remove config for dashboard nvmeof gateways if any
         ret, out, err = self.mgr.mon_command({
             'prefix': 'dashboard nvmeof-gateway-rm',
-            'name': daemon.hostname,
+            'name': service_name,
         })
         if not ret:
             logger.info(f'{daemon.hostname} removed from nvmeof gateways dashboard config')

src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2

@@ -41,10 +41,11 @@ config_file = /etc/ceph/ceph.conf
 id = {{ rados_id }}
 
 [mtls]
-server_key = {{ spec.server_key }}
-client_key = {{ spec.client_key }}
-server_cert = {{ spec.server_cert }}
-client_cert = {{ spec.client_cert }}
+server_key = /server.key
+client_key = /client.key
+server_cert = /server.cert
+client_cert = /client.cert
+root_ca_cert = /root.ca.cert
 
 [spdk]
 tgt_path = {{ spec.tgt_path }}