mgr/cephadm: adding mTLS support

Signed-off-by: Redouane Kachach <[email protected]>

Author: rkachach

src/cephadm/cephadmlib/daemons/mgmt_gateway.py

@@ -104,9 +104,22 @@ def create_daemon_dirs(self, data_dir: str, uid: int, gid: int) -> None:
             raise OSError('data_dir is not a directory: %s' % (data_dir))
         logger.info('Writing mgmt-gateway config...')
         config_dir = os.path.join(data_dir, 'etc/')
-        makedirs(config_dir, uid, gid, 0o755)
-        recursive_chown(config_dir, uid, gid)
-        populate_files(config_dir, self.files, uid, gid)
+        ssl_dir = os.path.join(data_dir, 'etc/ssl')
+        for ddir in [config_dir, ssl_dir]:
+            makedirs(ddir, uid, gid, 0o755)
+            recursive_chown(ddir, uid, gid)
+        conf_files = {
+            fname: content
+            for fname, content in self.files.items()
+            if fname.endswith('.conf')
+        }
+        cert_files = {
+            fname: content
+            for fname, content in self.files.items()
+            if fname.endswith('.crt') or fname.endswith('.key')
+        }
+        populate_files(config_dir, conf_files, uid, gid)
+        populate_files(ssl_dir, cert_files, uid, gid)
 
     def _get_container_mounts(self, data_dir: str) -> Dict[str, str]:
         mounts: Dict[str, str] = {}
@@ -152,23 +165,6 @@ def customize_container_mounts(
                 os.path.join(
                     data_dir, 'etc/nginx_external_server.conf'
                 ): '/etc/nginx_external_server.conf:Z',
-                os.path.join(
-                    data_dir, 'etc/nginx_internal.crt'
-                ): '/etc/nginx/ssl/nginx_internal.crt:Z',
-                os.path.join(
-                    data_dir, 'etc/nginx_internal.key'
-                ): '/etc/nginx/ssl/nginx_internal.key:Z',
+                os.path.join(data_dir, 'etc/ssl'): '/etc/nginx/ssl/',
             }
         )
-
-        if 'nginx.crt' in self.files:
-            mounts.update(
-                {
-                    os.path.join(
-                        data_dir, 'etc/nginx.crt'
-                    ): '/etc/nginx/ssl/nginx.crt:Z',
-                    os.path.join(
-                        data_dir, 'etc/nginx.key'
-                    ): '/etc/nginx/ssl/nginx.key:Z',
-                }
-            )

src/pybind/mgr/cephadm/cert_mgr.py

@@ -1,6 +1,5 @@
 
-from cephadm.ssl_cert_utils import SSLCerts
-from threading import Lock
+from cephadm.ssl_cert_utils import SSLCerts, SSLConfigException
 from typing import TYPE_CHECKING, Tuple, Union, List
 
 if TYPE_CHECKING:
@@ -13,31 +12,21 @@ class CertMgr:
     CEPHADM_ROOT_CA_KEY = 'cephadm_root_ca_key'
 
     def __init__(self, mgr: "CephadmOrchestrator", ip: str) -> None:
-        self.lock = Lock()
-        self.initialized = False
-        with self.lock:
-            if self.initialized:
-                return
-            self.initialized = True
-            self.mgr = mgr
-            self.ssl_certs: SSLCerts = SSLCerts()
-            old_cert = self.mgr.cert_key_store.get_cert(self.CEPHADM_ROOT_CA_CERT)
-            old_key = self.mgr.cert_key_store.get_key(self.CEPHADM_ROOT_CA_KEY)
-            if old_key and old_cert:
+        self.ssl_certs: SSLCerts = SSLCerts()
+        old_cert = mgr.cert_key_store.get_cert(self.CEPHADM_ROOT_CA_CERT)
+        old_key = mgr.cert_key_store.get_key(self.CEPHADM_ROOT_CA_KEY)
+        if old_key and old_cert:
+            try:
                 self.ssl_certs.load_root_credentials(old_cert, old_key)
-            else:
-                self.ssl_certs.generate_root_cert(ip)
-                self.mgr.cert_key_store.save_cert(self.CEPHADM_ROOT_CA_CERT, self.ssl_certs.get_root_cert())
-                self.mgr.cert_key_store.save_key(self.CEPHADM_ROOT_CA_KEY, self.ssl_certs.get_root_key())
+            except SSLConfigException:
+                raise Exception("Cannot load cephadm root CA certificates.")
+        else:
+            self.ssl_certs.generate_root_cert(ip)
+            mgr.cert_key_store.save_cert(self.CEPHADM_ROOT_CA_CERT, self.ssl_certs.get_root_cert())
+            mgr.cert_key_store.save_key(self.CEPHADM_ROOT_CA_KEY, self.ssl_certs.get_root_key())
 
     def get_root_ca(self) -> str:
-        with self.lock:
-            if self.initialized:
-                return self.ssl_certs.get_root_cert()
-        raise Exception("Not initialized")
-
-    def generate_cert(self, host_fqdn: Union[str, List[str]], node_ip: str) -> Tuple[str, str]:
-        with self.lock:
-            if self.initialized:
-                return self.ssl_certs.generate_cert(host_fqdn, node_ip)
-        raise Exception("Not initialized")
+        return self.ssl_certs.get_root_cert()
+
+    def generate_cert(self, host_fqdn: Union[str, List[str]], node_ip: Union[str, List[str]]) -> Tuple[str, str]:
+        return self.ssl_certs.generate_cert(host_fqdn, node_ip)

src/pybind/mgr/cephadm/http_server.py

@@ -31,7 +31,8 @@ def __init__(self, mgr: "CephadmOrchestrator") -> None:
         self.service_discovery = ServiceDiscovery(mgr)
         self.cherrypy_shutdown_event = threading.Event()
         self._service_discovery_port = self.mgr.service_discovery_port
-        self.secure_monitoring_stack = self.mgr.secure_monitoring_stack
+        security_enabled, mgmt_gw_enabled = self.mgr._get_security_config()
+        self.security_enabled = security_enabled
         super().__init__(target=self.run)
 
     def configure_cherrypy(self) -> None:
@@ -45,12 +46,13 @@ def configure(self) -> None:
         self.agent.configure()
         self.service_discovery.configure(self.mgr.service_discovery_port,
                                          self.mgr.get_mgr_ip(),
-                                         self.secure_monitoring_stack)
+                                         self.security_enabled)
 
     def config_update(self) -> None:
         self.service_discovery_port = self.mgr.service_discovery_port
-        if self.secure_monitoring_stack != self.mgr.secure_monitoring_stack:
-            self.secure_monitoring_stack = self.mgr.secure_monitoring_stack
+        security_enabled, mgmt_gw_enabled = self.mgr._get_security_config()
+        if self.security_enabled != security_enabled:
+            self.security_enabled = security_enabled
             self.restart()
 
     @property

src/pybind/mgr/cephadm/inventory.py

@@ -1942,16 +1942,15 @@ def _init_known_cert_key_dicts(self) -> None:
             'nvmeof_server_cert': {},  # service-name -> cert
             'nvmeof_client_cert': {},  # service-name -> cert
             'nvmeof_root_ca_cert': {},  # service-name -> cert
-            'agent_endpoint_root_cert': Cert(),  # cert
-            'mgmt_gw_root_cert': Cert(),  # cert
-            'service_discovery_root_cert': Cert(),  # cert
+            'mgmt_gw_cert': Cert(),  # cert
             'cephadm_root_ca_cert': Cert(),  # cert
             'grafana_cert': {},  # host -> cert
         }
         # Similar to certs but for priv keys. Entries in known_certs
         # that don't have a key here are probably certs in PEM format
         # so there is no need to store a separate key
         self.known_keys = {
+            'mgmt_gw_key': PrivKey(),  # cert
             'cephadm_root_ca_key': PrivKey(),  # cert
             'grafana_key': {},  # host -> key
             'iscsi_ssl_key': {},  # service-name -> key

src/pybind/mgr/cephadm/module.py

@@ -6,6 +6,7 @@
 import logging
 import re
 import shlex
+import socket
 from collections import defaultdict
 from configparser import ConfigParser
 from contextlib import contextmanager
@@ -771,6 +772,23 @@ def _get_cephadm_service(self, service_type: str) -> CephadmService:
         assert service_type in ServiceSpec.KNOWN_SERVICE_TYPES
         return self.cephadm_services[service_type]
 
+    def get_fqdn(self, hostname: str) -> str:
+        """Get a host's FQDN with its hostname.
+
+           If the FQDN can't be resolved, the address from the inventory will
+           be returned instead.
+        """
+        # TODO(redo): get fqdn from the inventory
+        addr = self.inventory.get_addr(hostname)
+        return socket.getfqdn(addr)
+
+    def _get_security_config(self) -> Tuple[bool, bool]:
+        # TODO(redo): enable when oauth2-proxy code is active
+        # oauth2_proxy_enabled = len(self.mgr.cache.get_daemons_by_service('oauth2-proxy')) > 0
+        mgmt_gw_enabled = len(self.cache.get_daemons_by_service('mgmt-gateway')) > 0
+        security_enabled = self.secure_monitoring_stack or mgmt_gw_enabled
+        return security_enabled, mgmt_gw_enabled
+
     def _get_cephadm_binary_path(self) -> str:
         import hashlib
         m = hashlib.sha256()
@@ -2611,9 +2629,6 @@ def remove_service(self, service_name: str, force: bool = False) -> str:
                 raise OrchestratorError(
                     f'If {service_name} is removed then the following OSDs will remain, --force to proceed anyway\n{msg}')
 
-        if service_name == 'mgmt-gateway':
-            self.set_module_option('secure_monitoring_stack', False)
-
         found = self.spec_store.rm(service_name)
         if found and service_name.startswith('osd.'):
             self.spec_store.finally_rm(service_name)
@@ -2943,21 +2958,26 @@ def get_daemon_names(daemons: List[str]) -> List[str]:
             # add dependency on ceph-exporter daemons
             deps += [d.name() for d in self.cache.get_daemons_by_service('ceph-exporter')]
             deps += [d.name() for d in self.cache.get_daemons_by_service('mgmt-gateway')]
-            if self.secure_monitoring_stack:
+            security_enabled, _ = self._get_security_config()
+            if security_enabled:
                 if prometheus_user and prometheus_password:
                     deps.append(f'{hash(prometheus_user + prometheus_password)}')
                 if alertmanager_user and alertmanager_password:
                     deps.append(f'{hash(alertmanager_user + alertmanager_password)}')
         elif daemon_type == 'grafana':
             deps += get_daemon_names(['prometheus', 'loki', 'mgmt-gateway'])
-            if self.secure_monitoring_stack and prometheus_user and prometheus_password:
+            security_enabled, _ = self._get_security_config()
+            if security_enabled and prometheus_user and prometheus_password:
                 deps.append(f'{hash(prometheus_user + prometheus_password)}')
         elif daemon_type == 'alertmanager':
             deps += get_daemon_names(['mgr', 'alertmanager', 'snmp-gateway', 'mgmt-gateway'])
-            if self.secure_monitoring_stack and alertmanager_user and alertmanager_password:
+            security_enabled, _ = self._get_security_config()
+            if security_enabled and alertmanager_user and alertmanager_password:
                 deps.append(f'{hash(alertmanager_user + alertmanager_password)}')
         elif daemon_type == 'promtail':
             deps += get_daemon_names(['loki'])
+        elif daemon_type in ['ceph-exporter', 'node-exporter']:
+            deps += get_daemon_names(['mgmt-gateway'])
         elif daemon_type == JaegerAgentService.TYPE:
             for dd in self.cache.get_daemons_by_type(JaegerCollectorService.TYPE):
                 assert dd.hostname is not None
@@ -2972,7 +2992,7 @@ def get_daemon_names(daemons: List[str]) -> List[str]:
             # this daemon type doesn't need deps mgmt
             pass
 
-        if daemon_type in ['prometheus', 'node-exporter', 'alertmanager', 'grafana', 'mgmt-gateway']:
+        if daemon_type in ['prometheus', 'node-exporter', 'alertmanager', 'grafana']:
             deps.append(f'secure_monitoring_stack:{self.secure_monitoring_stack}')
 
         return sorted(deps)
@@ -3088,10 +3108,17 @@ def _get_prometheus_credentials(self) -> Tuple[str, str]:
 
     @handle_orch_error
     def generate_certificates(self, module_name: str) -> Optional[Dict[str, str]]:
+        import socket
         supported_moduels = ['dashboard', 'prometheus']
         if module_name not in supported_moduels:
             raise OrchestratorError(f'Unsupported modlue {module_name}. Supported moduels are: {supported_moduels}')
-        cert, key = self.cert_mgr.generate_cert(self.get_hostname(), self.get_mgr_ip())
+
+        host_fqdns = [socket.getfqdn(self.get_hostname())]
+        node_ip = self.get_mgr_ip()
+        if module_name == 'dashboard':
+            host_fqdns.append('dashboard_servers')
+
+        cert, key = self.cert_mgr.generate_cert(host_fqdns, node_ip)
         return {'cert': cert, 'key': key}
 
     @handle_orch_error
@@ -3148,13 +3175,19 @@ def set_alertmanager_access_info(self, user: str, password: str) -> str:
 
     @handle_orch_error
     def get_prometheus_access_info(self) -> Dict[str, str]:
+        security_enabled, _ = self._get_security_config()
+        if not security_enabled:
+            return {}
         user, password = self._get_prometheus_credentials()
         return {'user': user,
                 'password': password,
                 'certificate': self.cert_mgr.get_root_ca()}
 
     @handle_orch_error
     def get_alertmanager_access_info(self) -> Dict[str, str]:
+        security_enabled, _ = self._get_security_config()
+        if not security_enabled:
+            return {}
         user, password = self._get_alertmanager_credentials()
         return {'user': user,
                 'password': password,
@@ -3403,9 +3436,6 @@ def _apply_service_spec(self, spec: ServiceSpec) -> str:
         host_count = len(self.inventory.keys())
         max_count = self.max_count_per_host
 
-        if spec.service_type == 'mgmt-gateway':
-            self.set_module_option('secure_monitoring_stack', True)
-
         if spec.placement.count is not None:
             if spec.service_type in ['mon', 'mgr']:
                 if spec.placement.count > max(5, host_count):

src/pybind/mgr/cephadm/services/cephadmservice.py

@@ -90,7 +90,7 @@ def get_dashboard_endpoints(svc: 'CephadmService') -> Tuple[List[str], Optional[
             if not port:
                 continue
             assert dd.hostname is not None
-            addr = svc._inventory_get_fqdn(dd.hostname)
+            addr = svc.mgr.get_fqdn(dd.hostname)
             dashboard_endpoints.append(f'{addr}:{port}')
 
     return dashboard_endpoints, protocol
@@ -124,7 +124,7 @@ def get_dashboard_urls(svc: 'CephadmService') -> List[str]:
         if dd.daemon_id == svc.mgr.get_mgr_id():
             continue
         assert dd.hostname is not None
-        addr = svc._inventory_get_fqdn(dd.hostname)
+        addr = svc.mgr.get_fqdn(dd.hostname)
         dashboard_urls.append(build_url(scheme=proto, host=addr, port=port).rstrip('/'))
 
     return dashboard_urls
@@ -384,15 +384,6 @@ def get_keyring_with_caps(self, entity: AuthEntity, caps: List[str]) -> str:
                 raise OrchestratorError(f"Unable to fetch keyring for {entity}: {err}")
         return simplified_keyring(entity, keyring)
 
-    def _inventory_get_fqdn(self, hostname: str) -> str:
-        """Get a host's FQDN with its hostname.
-
-           If the FQDN can't be resolved, the address from the inventory will
-           be returned instead.
-        """
-        addr = self.mgr.inventory.get_addr(hostname)
-        return socket.getfqdn(addr)
-
     def _set_value_on_dashboard(self,
                                 service_name: str,
                                 get_mon_cmd: str,
@@ -1282,11 +1273,29 @@ def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonD
         if spec.stats_period:
             exporter_config.update({'stats-period': f'{spec.stats_period}'})
 
+        security_enabled, mgmt_gw_enabled = self.mgr._get_security_config()
+        if security_enabled:
+            exporter_config.update({'https_enabled': True})
+            crt, key = self.get_certificates(daemon_spec)
+            exporter_config['files'] = {
+                'ceph-exporter.crt': crt,
+                'ceph-exporter.key': key
+            }
         daemon_spec.keyring = keyring
         daemon_spec.final_config, daemon_spec.deps = self.generate_config(daemon_spec)
         daemon_spec.final_config = merge_dicts(daemon_spec.final_config, exporter_config)
+
+        deps = []
+        deps += [d.name() for d in self.mgr.cache.get_daemons_by_service('mgmt-gateway')]
+        deps += [f'secure_monitoring_stack:{self.mgr.secure_monitoring_stack}']
+        daemon_spec.deps = deps
+
         return daemon_spec
 
+    def get_certificates(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[str, str]:
+        node_ip = self.mgr.inventory.get_addr(daemon_spec.host)
+        host_fqdn = self.mgr.get_fqdn(daemon_spec.host)
+        return self.mgr.cert_mgr.generate_cert(host_fqdn, node_ip)
 
 class CephfsMirrorService(CephService):
     TYPE = 'cephfs-mirror'