Merge pull request ceph#58402 from rkachach/fix_issue_mtls_support

mgr/cephadm: adding mTLS for ceph mgmt-gateway and backend services communication

Reviewed-by: Adam King <[email protected]>

Author: adk3798

src/cephadm/cephadm.py

@@ -167,6 +167,7 @@
 from cephadmlib.daemons.ceph import get_ceph_mounts_for_type, ceph_daemons
 from cephadmlib.daemons import (
     Ceph,
+    CephExporter,
     CephIscsi,
     CephNvmeof,
     CustomContainer,
@@ -867,6 +868,10 @@ def create_daemon_dirs(
         node_proxy = NodeProxy.init(ctx, fsid, ident.daemon_id)
         node_proxy.create_daemon_dirs(data_dir, uid, gid)
 
+    elif daemon_type == CephExporter.daemon_type:
+        ceph_exporter = CephExporter.init(ctx, fsid, ident.daemon_id)
+        ceph_exporter.create_daemon_dirs(data_dir, uid, gid)
+
     else:
         daemon = daemon_form_create(ctx, ident)
         if isinstance(daemon, ContainerDaemonForm):
@@ -2421,11 +2426,23 @@ def prepare_dashboard(
             pathify(ctx.dashboard_crt.name): '/tmp/dashboard.crt:z',
             pathify(ctx.dashboard_key.name): '/tmp/dashboard.key:z'
         }
-        cli(['dashboard', 'set-ssl-certificate', '-i', '/tmp/dashboard.crt'], extra_mounts=mounts)
-        cli(['dashboard', 'set-ssl-certificate-key', '-i', '/tmp/dashboard.key'], extra_mounts=mounts)
     else:
-        logger.info('Generating a dashboard self-signed certificate...')
-        cli(['dashboard', 'create-self-signed-cert'])
+        logger.info('Using certmgr to generate dashboard self-signed certificate...')
+        cert_key = json_loads_retry(lambda: cli(['orch', 'certmgr', 'generate-certificates', 'dashboard'],
+                                                verbosity=CallVerbosity.QUIET_UNLESS_ERROR))
+        mounts = {}
+        if cert_key:
+            cert_file = write_tmp(cert_key['cert'], uid, gid)
+            key_file = write_tmp(cert_key['key'], uid, gid)
+            mounts = {
+                cert_file.name: '/tmp/dashboard.crt:z',
+                key_file.name: '/tmp/dashboard.key:z'
+            }
+        else:
+            logger.error('Cannot generate certificates for Ceph dashboard.')
+
+    cli(['dashboard', 'set-ssl-certificate', '-i', '/tmp/dashboard.crt'], extra_mounts=mounts)
+    cli(['dashboard', 'set-ssl-certificate-key', '-i', '/tmp/dashboard.key'], extra_mounts=mounts)
 
     logger.info('Creating initial admin user...')
     password = ctx.initial_dashboard_password or generate_password()

src/cephadm/cephadmlib/daemons/ceph.py

@@ -16,7 +16,14 @@
 from ..context import CephadmContext
 from ..deployment_utils import to_deployment_container
 from ..exceptions import Error
-from ..file_utils import make_run_dir, pathify
+from ..file_utils import (
+    make_run_dir,
+    pathify,
+    populate_files,
+    makedirs,
+    recursive_chown,
+)
+from ..data_utils import dict_get
 from ..host_facts import HostFacts
 from ..logging import Highlight
 from ..net_utils import get_hostname, get_ip_addresses
@@ -298,6 +305,8 @@ def __init__(
         self.port = config_json.get('port', self.DEFAULT_PORT)
         self.prio_limit = config_json.get('prio-limit', 5)
         self.stats_period = config_json.get('stats-period', 5)
+        self.https_enabled: bool = config_json.get('https_enabled', False)
+        self.files = dict_get(config_json, 'files', {})
 
     @classmethod
     def init(
@@ -323,6 +332,15 @@ def get_daemon_args(self) -> List[str]:
             f'--prio-limit={self.prio_limit}',
             f'--stats-period={self.stats_period}',
         ]
+        if self.https_enabled:
+            args.extend(
+                [
+                    '--cert-file',
+                    '/etc/certs/ceph-exporter.crt',
+                    '--key-file',
+                    '/etc/certs/ceph-exporter.key',
+                ]
+            )
         return args
 
     def validate(self) -> None:
@@ -348,6 +366,9 @@ def customize_container_mounts(
     ) -> None:
         cm = Ceph.get_ceph_mounts(ctx, self.identity)
         mounts.update(cm)
+        if self.https_enabled:
+            data_dir = self.identity.data_dir(ctx.data_dir)
+            mounts.update({os.path.join(data_dir, 'etc/certs'): '/etc/certs'})
 
     def customize_process_args(
         self, ctx: CephadmContext, args: List[str]
@@ -376,6 +397,23 @@ def prepare_data_dir(self, data_dir: str, uid: int, gid: int) -> None:
         # it until now
         self.validate()
 
+    def create_daemon_dirs(self, data_dir: str, uid: int, gid: int) -> None:
+        """Create files under the container data dir"""
+        if not os.path.isdir(data_dir):
+            raise OSError('data_dir is not a directory: %s' % (data_dir))
+        logger.info('Writing ceph-exporter config...')
+        config_dir = os.path.join(data_dir, 'etc/')
+        ssl_dir = os.path.join(data_dir, 'etc/certs')
+        for ddir in [config_dir, ssl_dir]:
+            makedirs(ddir, uid, gid, 0o755)
+            recursive_chown(ddir, uid, gid)
+        cert_files = {
+            fname: content
+            for fname, content in self.files.items()
+            if fname.endswith('.crt') or fname.endswith('.key')
+        }
+        populate_files(ssl_dir, cert_files, uid, gid)
+
 
 def get_ceph_mounts_for_type(
     ctx: CephadmContext, fsid: str, daemon_type: str

src/cephadm/cephadmlib/daemons/mgmt_gateway.py

@@ -104,9 +104,22 @@ def create_daemon_dirs(self, data_dir: str, uid: int, gid: int) -> None:
             raise OSError('data_dir is not a directory: %s' % (data_dir))
         logger.info('Writing mgmt-gateway config...')
         config_dir = os.path.join(data_dir, 'etc/')
-        makedirs(config_dir, uid, gid, 0o755)
-        recursive_chown(config_dir, uid, gid)
-        populate_files(config_dir, self.files, uid, gid)
+        ssl_dir = os.path.join(data_dir, 'etc/ssl')
+        for ddir in [config_dir, ssl_dir]:
+            makedirs(ddir, uid, gid, 0o755)
+            recursive_chown(ddir, uid, gid)
+        conf_files = {
+            fname: content
+            for fname, content in self.files.items()
+            if fname.endswith('.conf')
+        }
+        cert_files = {
+            fname: content
+            for fname, content in self.files.items()
+            if fname.endswith('.crt') or fname.endswith('.key')
+        }
+        populate_files(config_dir, conf_files, uid, gid)
+        populate_files(ssl_dir, cert_files, uid, gid)
 
     def _get_container_mounts(self, data_dir: str) -> Dict[str, str]:
         mounts: Dict[str, str] = {}
@@ -152,23 +165,6 @@ def customize_container_mounts(
                 os.path.join(
                     data_dir, 'etc/nginx_external_server.conf'
                 ): '/etc/nginx_external_server.conf:Z',
-                os.path.join(
-                    data_dir, 'etc/nginx_internal.crt'
-                ): '/etc/nginx/ssl/nginx_internal.crt:Z',
-                os.path.join(
-                    data_dir, 'etc/nginx_internal.key'
-                ): '/etc/nginx/ssl/nginx_internal.key:Z',
+                os.path.join(data_dir, 'etc/ssl'): '/etc/nginx/ssl/',
             }
         )
-
-        if 'nginx.crt' in self.files:
-            mounts.update(
-                {
-                    os.path.join(
-                        data_dir, 'etc/nginx.crt'
-                    ): '/etc/nginx/ssl/nginx.crt:Z',
-                    os.path.join(
-                        data_dir, 'etc/nginx.key'
-                    ): '/etc/nginx/ssl/nginx.key:Z',
-                }
-            )

src/cephadm/tests/test_cephadm.py

@@ -282,7 +282,8 @@ def wrap_test(address, expected):
     @mock.patch('cephadmlib.firewalld.Firewalld', mock_bad_firewalld)
     @mock.patch('cephadm.Firewalld', mock_bad_firewalld)
     @mock.patch('cephadm.logger')
-    def test_skip_firewalld(self, _logger, cephadm_fs):
+    @mock.patch('cephadm.json_loads_retry', return_value=None)
+    def test_skip_firewalld(self, _logger, _jlr, cephadm_fs):
         """
         test --skip-firewalld actually skips changing firewall
         """

src/pybind/mgr/cephadm/agent.py

@@ -10,7 +10,6 @@ class Server:  # type: ignore
 import logging
 import socket
 import ssl
-import tempfile
 import threading
 import time
 
@@ -20,11 +19,12 @@ class Server:  # type: ignore
 from ceph.deployment.inventory import Devices
 from ceph.deployment.service_spec import ServiceSpec, PlacementSpec
 from cephadm.services.cephadmservice import CephadmDaemonDeploySpec
-from cephadm.ssl_cert_utils import SSLCerts
 from mgr_util import test_port_allocation, PortAlreadyInUse
+from mgr_util import verify_tls_files
+import tempfile
 
 from urllib.error import HTTPError, URLError
-from typing import Any, Dict, List, Set, TYPE_CHECKING, Optional, MutableMapping
+from typing import Any, Dict, List, Set, TYPE_CHECKING, Optional, MutableMapping, IO
 
 if TYPE_CHECKING:
     from cephadm.module import CephadmOrchestrator
@@ -46,9 +46,10 @@ class AgentEndpoint:
 
     def __init__(self, mgr: "CephadmOrchestrator") -> None:
         self.mgr = mgr
-        self.ssl_certs = SSLCerts()
         self.server_port = 7150
         self.server_addr = self.mgr.get_mgr_ip()
+        self.key_file: IO[bytes]
+        self.cert_file: IO[bytes]
 
     def configure_routes(self) -> None:
         conf = {'/': {'tools.trailing_slash.on': False}}
@@ -57,19 +58,19 @@ def configure_routes(self) -> None:
         cherrypy.tree.mount(self.node_proxy_endpoint, '/node-proxy', config=conf)
 
     def configure_tls(self, server: Server) -> None:
-        old_cert = self.mgr.cert_key_store.get_cert('agent_endpoint_root_cert')
-        old_key = self.mgr.cert_key_store.get_key('agent_endpoint_key')
+        addr = self.mgr.get_mgr_ip()
+        host = self.mgr.get_hostname()
+        cert, key = self.mgr.cert_mgr.generate_cert(host, addr)
+        self.cert_file = tempfile.NamedTemporaryFile()
+        self.cert_file.write(cert.encode('utf-8'))
+        self.cert_file.flush()  # cert_tmp must not be gc'ed
 
-        if old_cert and old_key:
-            self.ssl_certs.load_root_credentials(old_cert, old_key)
-        else:
-            self.ssl_certs.generate_root_cert(self.mgr.get_mgr_ip())
-            self.mgr.cert_key_store.save_cert('agent_endpoint_root_cert', self.ssl_certs.get_root_cert())
-            self.mgr.cert_key_store.save_key('agent_endpoint_key', self.ssl_certs.get_root_key())
+        self.key_file = tempfile.NamedTemporaryFile()
+        self.key_file.write(key.encode('utf-8'))
+        self.key_file.flush()  # pkey_tmp must not be gc'ed
 
-        host = self.mgr.get_hostname()
-        addr = self.mgr.get_mgr_ip()
-        server.ssl_certificate, server.ssl_private_key = self.ssl_certs.generate_cert_files(host, addr)
+        verify_tls_files(self.cert_file.name, self.key_file.name)
+        server.ssl_certificate, server.ssl_private_key = self.cert_file.name, self.key_file.name
 
     def find_free_port(self) -> None:
         max_port = self.server_port + 150
@@ -94,7 +95,7 @@ def configure(self) -> None:
 class NodeProxyEndpoint:
     def __init__(self, mgr: "CephadmOrchestrator"):
         self.mgr = mgr
-        self.ssl_root_crt = self.mgr.http_server.agent.ssl_certs.get_root_cert()
+        self.ssl_root_crt = self.mgr.cert_mgr.get_root_ca()
         self.ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
         self.ssl_ctx.check_hostname = False
         self.ssl_ctx.verify_mode = ssl.CERT_NONE
@@ -301,7 +302,7 @@ def led(self, **kw: Any) -> Dict[str, Any]:
         endpoint: List[Any] = ['led', led_type]
         device: str = id_drive if id_drive else ''
 
-        ssl_root_crt = self.mgr.http_server.agent.ssl_certs.get_root_cert()
+        ssl_root_crt = self.mgr.cert_mgr.get_root_ca()
         ssl_ctx = ssl.create_default_context()
         ssl_ctx.check_hostname = True
         ssl_ctx.verify_mode = ssl.CERT_REQUIRED
@@ -774,14 +775,13 @@ def run(self) -> None:
         self.mgr.agent_cache.sending_agent_message[self.host] = True
         try:
             assert self.agent
-            root_cert = self.agent.ssl_certs.get_root_cert()
+            root_cert = self.mgr.cert_mgr.get_root_ca()
             root_cert_tmp = tempfile.NamedTemporaryFile()
             root_cert_tmp.write(root_cert.encode('utf-8'))
             root_cert_tmp.flush()
             root_cert_fname = root_cert_tmp.name
 
-            cert, key = self.agent.ssl_certs.generate_cert(
-                self.mgr.get_hostname(), self.mgr.get_mgr_ip())
+            cert, key = self.mgr.cert_mgr.generate_cert(self.mgr.get_hostname(), self.mgr.get_mgr_ip())
 
             cert_tmp = tempfile.NamedTemporaryFile()
             cert_tmp.write(cert.encode('utf-8'))
@@ -950,7 +950,7 @@ def _check_agent(self, host: str) -> bool:
         down = False
         try:
             assert self.agent
-            assert self.agent.ssl_certs.get_root_cert()
+            assert self.mgr.cert_mgr.get_root_ca()
         except Exception:
             self.mgr.log.debug(
                 f'Delaying checking agent on {host} until cephadm endpoint finished creating root cert')
@@ -974,7 +974,7 @@ def _check_agent(self, host: str) -> bool:
                 # so it's necessary to check this one specifically
                 root_cert_match = False
                 try:
-                    root_cert = self.agent.ssl_certs.get_root_cert()
+                    root_cert = self.mgr.cert_mgr.get_root_ca()
                     if last_deps and root_cert in last_deps:
                         root_cert_match = True
                 except Exception:

src/pybind/mgr/cephadm/cert_mgr.py

@@ -0,0 +1,32 @@
+
+from cephadm.ssl_cert_utils import SSLCerts, SSLConfigException
+from typing import TYPE_CHECKING, Tuple, Union, List
+
+if TYPE_CHECKING:
+    from cephadm.module import CephadmOrchestrator
+
+
+class CertMgr:
+
+    CEPHADM_ROOT_CA_CERT = 'cephadm_root_ca_cert'
+    CEPHADM_ROOT_CA_KEY = 'cephadm_root_ca_key'
+
+    def __init__(self, mgr: "CephadmOrchestrator", ip: str) -> None:
+        self.ssl_certs: SSLCerts = SSLCerts()
+        old_cert = mgr.cert_key_store.get_cert(self.CEPHADM_ROOT_CA_CERT)
+        old_key = mgr.cert_key_store.get_key(self.CEPHADM_ROOT_CA_KEY)
+        if old_key and old_cert:
+            try:
+                self.ssl_certs.load_root_credentials(old_cert, old_key)
+            except SSLConfigException:
+                raise Exception("Cannot load cephadm root CA certificates.")
+        else:
+            self.ssl_certs.generate_root_cert(ip)
+            mgr.cert_key_store.save_cert(self.CEPHADM_ROOT_CA_CERT, self.ssl_certs.get_root_cert())
+            mgr.cert_key_store.save_key(self.CEPHADM_ROOT_CA_KEY, self.ssl_certs.get_root_key())
+
+    def get_root_ca(self) -> str:
+        return self.ssl_certs.get_root_cert()
+
+    def generate_cert(self, host_fqdn: Union[str, List[str]], node_ip: Union[str, List[str]]) -> Tuple[str, str]:
+        return self.ssl_certs.generate_cert(host_fqdn, node_ip)

src/pybind/mgr/cephadm/http_server.py

@@ -31,7 +31,8 @@ def __init__(self, mgr: "CephadmOrchestrator") -> None:
         self.service_discovery = ServiceDiscovery(mgr)
         self.cherrypy_shutdown_event = threading.Event()
         self._service_discovery_port = self.mgr.service_discovery_port
-        self.secure_monitoring_stack = self.mgr.secure_monitoring_stack
+        security_enabled, mgmt_gw_enabled = self.mgr._get_security_config()
+        self.security_enabled = security_enabled
         super().__init__(target=self.run)
 
     def configure_cherrypy(self) -> None:
@@ -45,12 +46,13 @@ def configure(self) -> None:
         self.agent.configure()
         self.service_discovery.configure(self.mgr.service_discovery_port,
                                          self.mgr.get_mgr_ip(),
-                                         self.secure_monitoring_stack)
+                                         self.security_enabled)
 
     def config_update(self) -> None:
         self.service_discovery_port = self.mgr.service_discovery_port
-        if self.secure_monitoring_stack != self.mgr.secure_monitoring_stack:
-            self.secure_monitoring_stack = self.mgr.secure_monitoring_stack
+        security_enabled, mgmt_gw_enabled = self.mgr._get_security_config()
+        if self.security_enabled != security_enabled:
+            self.security_enabled = security_enabled
             self.restart()
 
     @property