mgr/cephadm: add migration for certs/keys to cert store

Signed-off-by: Adam King <[email protected]>

Author: adk3798

src/pybind/mgr/cephadm/agent.py

@@ -44,10 +44,6 @@ def cherrypy_filter(record: logging.LogRecord) -> bool:
 
 class AgentEndpoint:
 
-    # TODO: move these constants to migrations
-    KV_STORE_AGENT_ROOT_CERT = 'cephadm_agent/root/cert'
-    KV_STORE_AGENT_ROOT_KEY = 'cephadm_agent/root/key'
-
     def __init__(self, mgr: "CephadmOrchestrator") -> None:
         self.mgr = mgr
         self.ssl_certs = SSLCerts()

src/pybind/mgr/cephadm/migrations.py

@@ -14,7 +14,7 @@
 if TYPE_CHECKING:
     from .module import CephadmOrchestrator
 
-LAST_MIGRATION = 6
+LAST_MIGRATION = 7
 
 logger = logging.getLogger(__name__)
 
@@ -105,6 +105,10 @@ def migrate(self, startup: bool = False) -> None:
             if self.migrate_5_6():
                 self.set(6)
 
+        if self.mgr.migration_current == 6:
+            if self.migrate_6_7():
+                self.set(7)
+
     def migrate_0_1(self) -> bool:
         """
         Migration 0 -> 1
@@ -410,6 +414,60 @@ def migrate_5_6(self) -> bool:
         self.rgw_migration_queue = []
         return True
 
+    def migrate_6_7(self) -> bool:
+        # start by placing certs/keys from rgw, iscsi, and ingress specs into cert store
+        for spec in self.mgr.spec_store.all_specs.values():
+            if spec.service_type in ['rgw', 'ingress', 'iscsi']:
+                logger.info(f'Migrating certs/keys for {spec.service_name()} spec to cert store')
+                self.mgr.spec_store._save_certs_and_keys(spec)
+
+        # Migrate service discovery and agent endpoint certs
+        # These constants were taken from where these certs were
+        # originally generated and should be the location they
+        # were store at prior to the cert store
+        KV_STORE_AGENT_ROOT_CERT = 'cephadm_agent/root/cert'
+        KV_STORE_AGENT_ROOT_KEY = 'cephadm_agent/root/key'
+        KV_STORE_SD_ROOT_CERT = 'service_discovery/root/cert'
+        KV_STORE_SD_ROOT_KEY = 'service_discovery/root/key'
+
+        agent_endpoint_cert = self.mgr.get_store(KV_STORE_AGENT_ROOT_CERT)
+        if agent_endpoint_cert:
+            logger.info('Migrating agent root cert to cert store')
+            self.mgr.cert_key_store.save_cert('agent_endpoint_root_cert', agent_endpoint_cert)
+        agent_endpoint_key = self.mgr.get_store(KV_STORE_AGENT_ROOT_KEY)
+        if agent_endpoint_key:
+            logger.info('Migrating agent root key to cert store')
+            self.mgr.cert_key_store.save_key('agent_endpoint_key', agent_endpoint_key)
+        service_discovery_cert = self.mgr.get_store(KV_STORE_SD_ROOT_CERT)
+        if service_discovery_cert:
+            logger.info('Migrating service discovery cert to cert store')
+            self.mgr.cert_key_store.save_cert('service_discovery_root_cert', service_discovery_cert)
+        service_discovery_key = self.mgr.get_store(KV_STORE_SD_ROOT_KEY)
+        if service_discovery_key:
+            logger.info('Migrating service discovery key to cert store')
+            self.mgr.cert_key_store.save_key('service_discovery_key', service_discovery_key)
+
+        # grafana certs are stored based on the host they are placed on
+        for grafana_daemon in self.mgr.cache.get_daemons_by_type('grafana'):
+            logger.info(f'Checking for cert/key for {grafana_daemon.name()}')
+            hostname = grafana_daemon.hostname
+            assert hostname is not None  # for mypy
+            grafana_cert_path = f'{hostname}/grafana_crt'
+            grafana_key_path = f'{hostname}/grafana_key'
+            grafana_cert = self.mgr.get_store(grafana_cert_path)
+            if grafana_cert:
+                logger.info(f'Migrating {grafana_daemon.name()} cert to cert store')
+                self.mgr.cert_key_store.save_cert('grafana_cert', grafana_cert, host=hostname)
+            grafana_key = self.mgr.get_store(grafana_key_path)
+            if grafana_key:
+                logger.info(f'Migrating {grafana_daemon.name()} key to cert store')
+                self.mgr.cert_key_store.save_key('grafana_key', grafana_key, host=hostname)
+
+        # NOTE: prometheus, alertmanager, and node-exporter certs were not stored
+        # and appeared to just be generated at daemon deploy time if secure_monitoring_stack
+        # was set to true. Therefore we have nothing to migrate for those daemons
+        return True
+
 
 def queue_migrate_rgw_spec(mgr: "CephadmOrchestrator", spec_dict: Dict[Any, Any]) -> None:
     """

src/pybind/mgr/cephadm/service_discovery.py

@@ -45,11 +45,6 @@ class Route(NamedTuple):
 
 class ServiceDiscovery:
 
-    # TODO: these constants should only be needed for migration purposes
-    # after completion of the cert store. Make sure to move them.
-    KV_STORE_SD_ROOT_CERT = 'service_discovery/root/cert'
-    KV_STORE_SD_ROOT_KEY = 'service_discovery/root/key'
-
     def __init__(self, mgr: "CephadmOrchestrator") -> None:
         self.mgr = mgr
         self.ssl_certs = SSLCerts()

src/pybind/mgr/cephadm/services/monitoring.py

@@ -121,9 +121,6 @@ def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[st
         return config_file, sorted(deps)
 
     def prepare_certificates(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[str, str]:
-        # TODO: move these variables to migrations
-        # cert_path = f'{daemon_spec.host}/grafana_crt'
-        # key_path = f'{daemon_spec.host}/grafana_key'
         cert = self.mgr.cert_key_store.get_cert('grafana_cert', host=daemon_spec.host)
         pkey = self.mgr.cert_key_store.get_key('grafana_key', host=daemon_spec.host)
         certs_present = (cert and pkey)

src/pybind/mgr/cephadm/tests/test_migration.py

@@ -1,13 +1,21 @@
 import json
 import pytest
 
-from ceph.deployment.service_spec import PlacementSpec, ServiceSpec, HostPlacementSpec
+from ceph.deployment.service_spec import (
+    PlacementSpec,
+    ServiceSpec,
+    HostPlacementSpec,
+    RGWSpec,
+    IngressSpec,
+    IscsiServiceSpec
+)
 from ceph.utils import datetime_to_str, datetime_now
 from cephadm import CephadmOrchestrator
 from cephadm.inventory import SPEC_STORE_PREFIX
 from cephadm.migrations import LAST_MIGRATION
 from cephadm.tests.fixtures import _run_cephadm, wait, with_host, receive_agent_metadata_all_hosts
 from cephadm.serve import CephadmServe
+from orchestrator import DaemonDescription
 from tests import mock
 
 
@@ -338,3 +346,44 @@ def test_migrate_rgw_spec(cephadm_module: CephadmOrchestrator, rgw_spec_store_en
             # if it was migrated, so we can use this to test the spec
             # was untouched
             assert 'rgw.foo' not in cephadm_module.spec_store.all_specs
+
+
+def test_migrate_cert_store(cephadm_module: CephadmOrchestrator):
+    rgw_spec = RGWSpec(service_id='foo', rgw_frontend_ssl_certificate='rgw_cert', ssl=True)
+    iscsi_spec = IscsiServiceSpec(service_id='foo', pool='foo', ssl_cert='iscsi_cert', ssl_key='iscsi_key')
+    ingress_spec = IngressSpec(service_id='rgw.foo', ssl_cert='ingress_cert', ssl_key='ingress_key', ssl=True)
+    cephadm_module.spec_store._specs = {
+        'rgw.foo': rgw_spec,
+        'iscsi.foo': iscsi_spec,
+        'ingress.rgw.foo': ingress_spec
+    }
+
+    cephadm_module.set_store('cephadm_agent/root/cert', 'agent_cert')
+    cephadm_module.set_store('cephadm_agent/root/key', 'agent_key')
+    cephadm_module.set_store('service_discovery/root/cert', 'service_discovery_cert')
+    cephadm_module.set_store('service_discovery/root/key', 'service_discovery_key')
+
+    cephadm_module.set_store('host1/grafana_crt', 'grafana_cert1')
+    cephadm_module.set_store('host1/grafana_key', 'grafana_key1')
+    cephadm_module.set_store('host2/grafana_crt', 'grafana_cert2')
+    cephadm_module.set_store('host2/grafana_key', 'grafana_key2')
+    cephadm_module.cache.daemons = {'host1': {'grafana.host1': DaemonDescription('grafana', 'host1', 'host1')},
+                                    'host2': {'grafana.host2': DaemonDescription('grafana', 'host2', 'host2')}}
+
+    cephadm_module.migration.migrate_6_7()
+
+    assert cephadm_module.cert_key_store.get_cert('rgw_frontend_ssl_cert', service_name='rgw.foo')
+    assert cephadm_module.cert_key_store.get_cert('iscsi_ssl_cert', service_name='iscsi.foo')
+    assert cephadm_module.cert_key_store.get_key('iscsi_ssl_key', service_name='iscsi.foo')
+    assert cephadm_module.cert_key_store.get_cert('ingress_ssl_cert', service_name='ingress.rgw.foo')
+    assert cephadm_module.cert_key_store.get_key('ingress_ssl_key', service_name='ingress.rgw.foo')
+
+    assert cephadm_module.cert_key_store.get_cert('agent_endpoint_root_cert')
+    assert cephadm_module.cert_key_store.get_key('agent_endpoint_key')
+    assert cephadm_module.cert_key_store.get_cert('service_discovery_root_cert')
+    assert cephadm_module.cert_key_store.get_key('service_discovery_key')
+
+    assert cephadm_module.cert_key_store.get_cert('grafana_cert', host='host1')
+    assert cephadm_module.cert_key_store.get_cert('grafana_cert', host='host2')
+    assert cephadm_module.cert_key_store.get_key('grafana_key', host='host1')
+    assert cephadm_module.cert_key_store.get_key('grafana_key', host='host2')