mgr/cephadm: introducing cert_mgr new class to centralize certs mgmt

cert_mgr will be the unique responsible of managing all certificates
generated and maintained by cephadm. Cephadm in addition now provides
a new cmd to generate certificates for external modules.

Signed-off-by: Redouane Kachach <[email protected]>

Author: rkachach

src/pybind/mgr/cephadm/agent.py

@@ -10,7 +10,6 @@ class Server:  # type: ignore
 import logging
 import socket
 import ssl
-import tempfile
 import threading
 import time
 
@@ -20,11 +19,12 @@ class Server:  # type: ignore
 from ceph.deployment.inventory import Devices
 from ceph.deployment.service_spec import ServiceSpec, PlacementSpec
 from cephadm.services.cephadmservice import CephadmDaemonDeploySpec
-from cephadm.ssl_cert_utils import SSLCerts
 from mgr_util import test_port_allocation, PortAlreadyInUse
+from mgr_util import verify_tls_files
+import tempfile
 
 from urllib.error import HTTPError, URLError
-from typing import Any, Dict, List, Set, TYPE_CHECKING, Optional, MutableMapping
+from typing import Any, Dict, List, Set, TYPE_CHECKING, Optional, MutableMapping, IO
 
 if TYPE_CHECKING:
     from cephadm.module import CephadmOrchestrator
@@ -46,9 +46,10 @@ class AgentEndpoint:
 
     def __init__(self, mgr: "CephadmOrchestrator") -> None:
         self.mgr = mgr
-        self.ssl_certs = SSLCerts()
         self.server_port = 7150
         self.server_addr = self.mgr.get_mgr_ip()
+        self.key_file: IO[bytes]
+        self.cert_file: IO[bytes]
 
     def configure_routes(self) -> None:
         conf = {'/': {'tools.trailing_slash.on': False}}
@@ -57,19 +58,19 @@ def configure_routes(self) -> None:
         cherrypy.tree.mount(self.node_proxy_endpoint, '/node-proxy', config=conf)
 
     def configure_tls(self, server: Server) -> None:
-        old_cert = self.mgr.cert_key_store.get_cert('agent_endpoint_root_cert')
-        old_key = self.mgr.cert_key_store.get_key('agent_endpoint_key')
+        addr = self.mgr.get_mgr_ip()
+        host = self.mgr.get_hostname()
+        cert, key = self.mgr.cert_mgr.generate_cert(host, addr)
+        self.cert_file = tempfile.NamedTemporaryFile()
+        self.cert_file.write(cert.encode('utf-8'))
+        self.cert_file.flush()  # cert_tmp must not be gc'ed
 
-        if old_cert and old_key:
-            self.ssl_certs.load_root_credentials(old_cert, old_key)
-        else:
-            self.ssl_certs.generate_root_cert(self.mgr.get_mgr_ip())
-            self.mgr.cert_key_store.save_cert('agent_endpoint_root_cert', self.ssl_certs.get_root_cert())
-            self.mgr.cert_key_store.save_key('agent_endpoint_key', self.ssl_certs.get_root_key())
+        self.key_file = tempfile.NamedTemporaryFile()
+        self.key_file.write(key.encode('utf-8'))
+        self.key_file.flush()  # pkey_tmp must not be gc'ed
 
-        host = self.mgr.get_hostname()
-        addr = self.mgr.get_mgr_ip()
-        server.ssl_certificate, server.ssl_private_key = self.ssl_certs.generate_cert_files(host, addr)
+        verify_tls_files(self.cert_file.name, self.key_file.name)
+        server.ssl_certificate, server.ssl_private_key = self.cert_file.name, self.key_file.name
 
     def find_free_port(self) -> None:
         max_port = self.server_port + 150
@@ -94,7 +95,7 @@ def configure(self) -> None:
 class NodeProxyEndpoint:
     def __init__(self, mgr: "CephadmOrchestrator"):
         self.mgr = mgr
-        self.ssl_root_crt = self.mgr.http_server.agent.ssl_certs.get_root_cert()
+        self.ssl_root_crt = self.mgr.cert_mgr.get_root_ca()
         self.ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
         self.ssl_ctx.check_hostname = False
         self.ssl_ctx.verify_mode = ssl.CERT_NONE
@@ -301,7 +302,7 @@ def led(self, **kw: Any) -> Dict[str, Any]:
         endpoint: List[Any] = ['led', led_type]
         device: str = id_drive if id_drive else ''
 
-        ssl_root_crt = self.mgr.http_server.agent.ssl_certs.get_root_cert()
+        ssl_root_crt = self.mgr.cert_mgr.get_root_ca()
         ssl_ctx = ssl.create_default_context()
         ssl_ctx.check_hostname = True
         ssl_ctx.verify_mode = ssl.CERT_REQUIRED
@@ -774,14 +775,13 @@ def run(self) -> None:
         self.mgr.agent_cache.sending_agent_message[self.host] = True
         try:
             assert self.agent
-            root_cert = self.agent.ssl_certs.get_root_cert()
+            root_cert = self.mgr.cert_mgr.get_root_ca()
             root_cert_tmp = tempfile.NamedTemporaryFile()
             root_cert_tmp.write(root_cert.encode('utf-8'))
             root_cert_tmp.flush()
             root_cert_fname = root_cert_tmp.name
 
-            cert, key = self.agent.ssl_certs.generate_cert(
-                self.mgr.get_hostname(), self.mgr.get_mgr_ip())
+            cert, key = self.mgr.cert_mgr.generate_cert(self.mgr.get_hostname(), self.mgr.get_mgr_ip())
 
             cert_tmp = tempfile.NamedTemporaryFile()
             cert_tmp.write(cert.encode('utf-8'))
@@ -950,7 +950,7 @@ def _check_agent(self, host: str) -> bool:
         down = False
         try:
             assert self.agent
-            assert self.agent.ssl_certs.get_root_cert()
+            assert self.mgr.cert_mgr.get_root_ca()
         except Exception:
             self.mgr.log.debug(
                 f'Delaying checking agent on {host} until cephadm endpoint finished creating root cert')
@@ -974,7 +974,7 @@ def _check_agent(self, host: str) -> bool:
                 # so it's necessary to check this one specifically
                 root_cert_match = False
                 try:
-                    root_cert = self.agent.ssl_certs.get_root_cert()
+                    root_cert = self.mgr.cert_mgr.get_root_ca()
                     if last_deps and root_cert in last_deps:
                         root_cert_match = True
                 except Exception:

src/pybind/mgr/cephadm/cert_mgr.py

@@ -0,0 +1,43 @@
+
+from cephadm.ssl_cert_utils import SSLCerts
+from threading import Lock
+from typing import TYPE_CHECKING, Tuple, Union, List
+
+if TYPE_CHECKING:
+    from cephadm.module import CephadmOrchestrator
+
+
+class CertMgr:
+
+    CEPHADM_ROOT_CA_CERT = 'cephadm_root_ca_cert'
+    CEPHADM_ROOT_CA_KEY = 'cephadm_root_ca_key'
+
+    def __init__(self, mgr: "CephadmOrchestrator", ip: str) -> None:
+        self.lock = Lock()
+        self.initialized = False
+        with self.lock:
+            if self.initialized:
+                return
+            self.initialized = True
+            self.mgr = mgr
+            self.ssl_certs: SSLCerts = SSLCerts()
+            old_cert = self.mgr.cert_key_store.get_cert(self.CEPHADM_ROOT_CA_CERT)
+            old_key = self.mgr.cert_key_store.get_key(self.CEPHADM_ROOT_CA_KEY)
+            if old_key and old_cert:
+                self.ssl_certs.load_root_credentials(old_cert, old_key)
+            else:
+                self.ssl_certs.generate_root_cert(ip)
+                self.mgr.cert_key_store.save_cert(self.CEPHADM_ROOT_CA_CERT, self.ssl_certs.get_root_cert())
+                self.mgr.cert_key_store.save_key(self.CEPHADM_ROOT_CA_KEY, self.ssl_certs.get_root_key())
+
+    def get_root_ca(self) -> str:
+        with self.lock:
+            if self.initialized:
+                return self.ssl_certs.get_root_cert()
+        raise Exception("Not initialized")
+
+    def generate_cert(self, host_fqdn: Union[str, List[str]], node_ip: str) -> Tuple[str, str]:
+        with self.lock:
+            if self.initialized:
+                return self.ssl_certs.generate_cert(host_fqdn, node_ip)
+        raise Exception("Not initialized")

src/pybind/mgr/cephadm/inventory.py

@@ -1912,16 +1912,10 @@ class CertKeyStore():
 
     host_cert = [
         'grafana_cert',
-        'alertmanager_cert',
-        'prometheus_cert',
-        'node_exporter_cert',
     ]
 
     host_key = [
         'grafana_key',
-        'alertmanager_key',
-        'prometheus_key',
-        'node_exporter_key',
     ]
 
     service_name_key = [
@@ -1951,22 +1945,15 @@ def _init_known_cert_key_dicts(self) -> None:
             'agent_endpoint_root_cert': Cert(),  # cert
             'mgmt_gw_root_cert': Cert(),  # cert
             'service_discovery_root_cert': Cert(),  # cert
+            'cephadm_root_ca_cert': Cert(),  # cert
             'grafana_cert': {},  # host -> cert
-            'alertmanager_cert': {},  # host -> cert
-            'prometheus_cert': {},  # host -> cert
-            'node_exporter_cert': {},  # host -> cert
         }
         # Similar to certs but for priv keys. Entries in known_certs
         # that don't have a key here are probably certs in PEM format
         # so there is no need to store a separate key
         self.known_keys = {
-            'agent_endpoint_key': PrivKey(),  # key
-            'service_discovery_key': PrivKey(),  # key
-            'mgmt_gw_root_key': PrivKey(),  # cert
+            'cephadm_root_ca_key': PrivKey(),  # cert
             'grafana_key': {},  # host -> key
-            'alertmanager_key': {},  # host -> key
-            'prometheus_key': {},  # host -> key
-            'node_exporter_key': {},  # host -> key
             'iscsi_ssl_key': {},  # service-name -> key
             'ingress_ssl_key': {},  # service-name -> key
             'nvmeof_server_key': {},  # service-name -> key

src/pybind/mgr/cephadm/migrations.py

@@ -421,32 +421,6 @@ def migrate_6_7(self) -> bool:
                 logger.info(f'Migrating certs/keys for {spec.service_name()} spec to cert store')
                 self.mgr.spec_store._save_certs_and_keys(spec)
 
-        # Migrate service discovery and agent endpoint certs
-        # These constants were taken from where these certs were
-        # originally generated and should be the location they
-        # were store at prior to the cert store
-        KV_STORE_AGENT_ROOT_CERT = 'cephadm_agent/root/cert'
-        KV_STORE_AGENT_ROOT_KEY = 'cephadm_agent/root/key'
-        KV_STORE_SD_ROOT_CERT = 'service_discovery/root/cert'
-        KV_STORE_SD_ROOT_KEY = 'service_discovery/root/key'
-
-        agent_endpoint_cert = self.mgr.get_store(KV_STORE_AGENT_ROOT_CERT)
-        if agent_endpoint_cert:
-            logger.info('Migrating agent root cert to cert store')
-            self.mgr.cert_key_store.save_cert('agent_endpoint_root_cert', agent_endpoint_cert)
-        agent_endpoint_key = self.mgr.get_store(KV_STORE_AGENT_ROOT_KEY)
-        if agent_endpoint_key:
-            logger.info('Migrating agent root key to cert store')
-            self.mgr.cert_key_store.save_key('agent_endpoint_key', agent_endpoint_key)
-        service_discovery_cert = self.mgr.get_store(KV_STORE_SD_ROOT_CERT)
-        if service_discovery_cert:
-            logger.info('Migrating service discovery cert to cert store')
-            self.mgr.cert_key_store.save_cert('service_discovery_root_cert', service_discovery_cert)
-        service_discovery_key = self.mgr.get_store(KV_STORE_SD_ROOT_KEY)
-        if service_discovery_key:
-            logger.info('Migrating service discovery key to cert store')
-            self.mgr.cert_key_store.save_key('service_discovery_key', service_discovery_key)
-
         # grafana certs are stored based on the host they are placed on
         for grafana_daemon in self.mgr.cache.get_daemons_by_type('grafana'):
             logger.info(f'Checking for cert/key for {grafana_daemon.name()}')

src/pybind/mgr/cephadm/module.py

@@ -15,6 +15,7 @@
 from threading import Event
 
 from ceph.deployment.service_spec import PrometheusSpec
+from cephadm.cert_mgr import CertMgr
 
 import string
 from typing import List, Dict, Optional, Callable, Tuple, TypeVar, \
@@ -538,11 +539,11 @@ def __init__(self, *args: Any, **kwargs: Any):
         super(CephadmOrchestrator, self).__init__(*args, **kwargs)
         self._cluster_fsid: str = self.get('mon_map')['fsid']
         self.last_monmap: Optional[datetime.datetime] = None
+        self.cert_mgr = CertMgr(self, self.get_mgr_ip())
 
         # for serve()
         self.run = True
         self.event = Event()
-
         self.ssh = ssh.SSHManager(self)
 
         if self.get_store('pause'):
@@ -2609,6 +2610,9 @@ def remove_service(self, service_name: str, force: bool = False) -> str:
                 raise OrchestratorError(
                     f'If {service_name} is removed then the following OSDs will remain, --force to proceed anyway\n{msg}')
 
+        if service_name == 'mgmt-gateway':
+            self.set_module_option('secure_monitoring_stack', False)
+
         found = self.spec_store.rm(service_name)
         if found and service_name.startswith('osd.'):
             self.spec_store.finally_rm(service_name)
@@ -2899,7 +2903,7 @@ def get_daemon_names(daemons: List[str]) -> List[str]:
             server_port = ''
             try:
                 server_port = str(self.http_server.agent.server_port)
-                root_cert = self.http_server.agent.ssl_certs.get_root_cert()
+                root_cert = self.cert_mgr.get_root_ca()
             except Exception:
                 pass
             deps = sorted([self.get_mgr_ip(), server_port, root_cert,
@@ -2909,7 +2913,7 @@ def get_daemon_names(daemons: List[str]) -> List[str]:
             server_port = ''
             try:
                 server_port = str(self.http_server.agent.server_port)
-                root_cert = self.http_server.agent.ssl_certs.get_root_cert()
+                root_cert = self.cert_mgr.get_root_ca()
             except Exception:
                 pass
             deps = sorted([self.get_mgr_ip(), server_port, root_cert])
@@ -3138,14 +3142,14 @@ def get_prometheus_access_info(self) -> Dict[str, str]:
         user, password = self._get_prometheus_credentials()
         return {'user': user,
                 'password': password,
-                'certificate': self.http_server.service_discovery.ssl_certs.get_root_cert()}
+                'certificate': self.cert_mgr.get_root_ca()}
 
     @handle_orch_error
     def get_alertmanager_access_info(self) -> Dict[str, str]:
         user, password = self._get_alertmanager_credentials()
         return {'user': user,
                 'password': password,
-                'certificate': self.http_server.service_discovery.ssl_certs.get_root_cert()}
+                'certificate': self.cert_mgr.get_root_ca()}
 
     @handle_orch_error
     def cert_store_cert_ls(self) -> Dict[str, Any]:
@@ -3397,6 +3401,9 @@ def _apply_service_spec(self, spec: ServiceSpec) -> str:
         host_count = len(self.inventory.keys())
         max_count = self.max_count_per_host
 
+        if spec.service_type == 'mgmt-gateway':
+            self.set_module_option('secure_monitoring_stack', True)
+
         if spec.placement.count is not None:
             if spec.service_type in ['mon', 'mgr']:
                 if spec.placement.count > max(5, host_count):

src/pybind/mgr/cephadm/serve.py

@@ -702,7 +702,7 @@ def _apply_service(self, spec: ServiceSpec) -> bool:
         if service_type == 'agent':
             try:
                 assert self.mgr.http_server.agent
-                assert self.mgr.http_server.agent.ssl_certs.get_root_cert()
+                assert self.mgr.cert_mgr.get_root_ca()
             except Exception:
                 self.log.info(
                     'Delaying applying agent spec until cephadm endpoint root cert created')

src/pybind/mgr/cephadm/service_discovery.py

@@ -7,17 +7,17 @@ class Server:  # type: ignore
         pass
 
 import logging
-import socket
 
 import orchestrator  # noqa
 from mgr_module import ServiceInfoT
 from mgr_util import build_url
-from typing import Dict, List, TYPE_CHECKING, cast, Collection, Callable, NamedTuple, Optional
+from typing import Dict, List, TYPE_CHECKING, cast, Collection, Callable, NamedTuple, Optional, IO
 from cephadm.services.monitoring import AlertmanagerService, NodeExporterService, PrometheusService
 import secrets
+from mgr_util import verify_tls_files
+import tempfile
 
 from cephadm.services.ingress import IngressSpec
-from cephadm.ssl_cert_utils import SSLCerts
 from cephadm.services.cephadmservice import CephExporterService
 from cephadm.services.nvmeof import NvmeofService
 
@@ -47,9 +47,10 @@ class ServiceDiscovery:
 
     def __init__(self, mgr: "CephadmOrchestrator") -> None:
         self.mgr = mgr
-        self.ssl_certs = SSLCerts()
         self.username: Optional[str] = None
         self.password: Optional[str] = None
+        self.key_file: IO[bytes]
+        self.cert_file: IO[bytes]
 
     def validate_password(self, realm: str, username: str, password: str) -> bool:
         return (password == self.password and username == self.username)
@@ -86,18 +87,20 @@ def enable_auth(self) -> None:
             self.mgr.set_store('service_discovery/root/username', self.username)
 
     def configure_tls(self, server: Server) -> None:
-        old_cert = self.mgr.cert_key_store.get_cert('service_discovery_root_cert')
-        old_key = self.mgr.cert_key_store.get_key('service_discovery_key')
-        if old_key and old_cert:
-            self.ssl_certs.load_root_credentials(old_cert, old_key)
-        else:
-            self.ssl_certs.generate_root_cert(self.mgr.get_mgr_ip())
-            self.mgr.cert_key_store.save_cert('service_discovery_root_cert', self.ssl_certs.get_root_cert())
-            self.mgr.cert_key_store.save_key('service_discovery_key', self.ssl_certs.get_root_key())
         addr = self.mgr.get_mgr_ip()
-        host_fqdn = socket.getfqdn(addr)
-        server.ssl_certificate, server.ssl_private_key = self.ssl_certs.generate_cert_files(
-            host_fqdn, addr)
+        host = self.mgr.get_hostname()
+        cert, key = self.mgr.cert_mgr.generate_cert(host, addr)
+        self.cert_file = tempfile.NamedTemporaryFile()
+        self.cert_file.write(cert.encode('utf-8'))
+        self.cert_file.flush()  # cert_tmp must not be gc'ed
+
+        self.key_file = tempfile.NamedTemporaryFile()
+        self.key_file.write(key.encode('utf-8'))
+        self.key_file.flush()  # pkey_tmp must not be gc'ed
+
+        verify_tls_files(self.cert_file.name, self.key_file.name)
+
+        server.ssl_certificate, server.ssl_private_key = self.cert_file.name, self.key_file.name
 
     def configure(self, port: int, addr: str, enable_security: bool) -> None:
         # we create a new server to enforce TLS/SSL config refresh