mgr/cephadm: sign generated RGW certs

Previously the "generate_cert" field would just cause cephadm
to generate self-signed certificates. This was an issue when trying to sync
the secondary site in a multisite situation, resulting in

```
SL peer certificate or SSH remote key was not OK req_data->error_buf=SSL
certificate problem: self-signed certificate request failed: (2200) Unknown error 2200
```

This change makes it so the certificate are signed by cephadm's
root CA cert so that users may grab that cert via
"ceph orch cert-store get cert cephadm_root_ca_cert"
and set that as a trusted CA cert on their secondary cluster.

Additionally, we now generate a cert per RGW daemon so that
we can include the hostname/addr of the node we are deploying
the RGW daemon on in the cert.

Signed-off-by: Adam King <[email protected]>

Author: adk3798

src/pybind/mgr/cephadm/cert_mgr.py

@@ -1,6 +1,6 @@
 
 from cephadm.ssl_cert_utils import SSLCerts, SSLConfigException
-from typing import TYPE_CHECKING, Tuple, Union, List
+from typing import TYPE_CHECKING, Tuple, Union, List, Optional
 
 if TYPE_CHECKING:
     from cephadm.module import CephadmOrchestrator
@@ -28,5 +28,10 @@ def __init__(self, mgr: "CephadmOrchestrator", ip: str) -> None:
     def get_root_ca(self) -> str:
         return self.ssl_certs.get_root_cert()
 
-    def generate_cert(self, host_fqdn: Union[str, List[str]], node_ip: Union[str, List[str]]) -> Tuple[str, str]:
-        return self.ssl_certs.generate_cert(host_fqdn, node_ip)
+    def generate_cert(
+        self,
+        host_fqdn: Union[str, List[str]],
+        node_ip: Union[str, List[str]],
+        custom_san_list: Optional[List[str]] = None,
+    ) -> Tuple[str, str]:
+        return self.ssl_certs.generate_cert(host_fqdn, node_ip, custom_san_list=custom_san_list)

src/pybind/mgr/cephadm/services/cephadmservice.py

@@ -1015,12 +1015,6 @@ def config(self, spec: RGWSpec) -> None:  # type: ignore
         # set rgw_realm rgw_zonegroup and rgw_zone, if present
         self.set_realm_zg_zone(spec)
 
-        if spec.generate_cert and not spec.rgw_frontend_ssl_certificate:
-            # generate a self-signed cert for the rgw service
-            cert, key = self.mgr.cert_mgr.ssl_certs.generate_root_cert(custom_san_list=spec.zonegroup_hostnames)
-            spec.rgw_frontend_ssl_certificate = ''.join([key, cert])
-            self.mgr.spec_store.save(spec)
-
         if spec.rgw_frontend_ssl_certificate:
             if isinstance(spec.rgw_frontend_ssl_certificate, list):
                 cert_data = '\n'.join(spec.rgw_frontend_ssl_certificate)
@@ -1068,6 +1062,19 @@ def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonD
             # and it matches the spec.
             port = spec.get_port()
 
+        if spec.generate_cert:
+            cert, key = self.mgr.cert_mgr.generate_cert(
+                daemon_spec.host,
+                self.mgr.inventory.get_addr(daemon_spec.host),
+                custom_san_list=spec.zonegroup_hostnames
+            )
+            pem = ''.join([key, cert])
+            ret, out, err = self.mgr.check_mon_command({
+                'prefix': 'config-key set',
+                'key': f'rgw/cert/{daemon_spec.name()}',
+                'val': pem,
+            })
+
         # configure frontend
         args = []
         ftype = spec.rgw_frontend_type or "beast"
@@ -1078,7 +1085,10 @@ def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonD
                         f"ssl_endpoint={build_url(host=daemon_spec.ip, port=port).lstrip('/')}")
                 else:
                     args.append(f"ssl_port={port}")
-                args.append(f"ssl_certificate=config://rgw/cert/{spec.service_name()}")
+                if spec.generate_cert:
+                    args.append(f"ssl_certificate=config://rgw/cert/{daemon_spec.name()}")
+                else:
+                    args.append(f"ssl_certificate=config://rgw/cert/{spec.service_name()}")
             else:
                 if daemon_spec.ip:
                     args.append(f"endpoint={build_url(host=daemon_spec.ip, port=port).lstrip('/')}")
@@ -1091,7 +1101,10 @@ def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonD
                     args.append(f"port={build_url(host=daemon_spec.ip, port=port).lstrip('/')}s")
                 else:
                     args.append(f"port={port}s")  # note the 's' suffix on port
-                args.append(f"ssl_certificate=config://rgw/cert/{spec.service_name()}")
+                if spec.generate_cert:
+                    args.append(f"ssl_certificate=config://rgw/cert/{daemon_spec.name()}")
+                else:
+                    args.append(f"ssl_certificate=config://rgw/cert/{spec.service_name()}")
             else:
                 if daemon_spec.ip:
                     args.append(f"port={build_url(host=daemon_spec.ip, port=port).lstrip('/')}")
@@ -1180,6 +1193,10 @@ def post_remove(self, daemon: DaemonDescription, is_failed_deploy: bool) -> None
             'who': utils.name_to_config_section(daemon.name()),
             'name': 'rgw_frontends',
         })
+        self.mgr.check_mon_command({
+            'prefix': 'config-key rm',
+            'key': f'rgw/cert/{daemon.name()}',
+        })
 
     def ok_to_stop(
             self,

src/pybind/mgr/cephadm/ssl_cert_utils.py

@@ -70,7 +70,12 @@ def generate_root_cert(
 
         return (cert_str, key_str)
 
-    def generate_cert(self, _hosts: Union[str, List[str]], _addrs: Union[str, List[str]]) -> Tuple[str, str]:
+    def generate_cert(
+        self,
+        _hosts: Union[str, List[str]],
+        _addrs: Union[str, List[str]],
+        custom_san_list: Optional[List[str]] = None,
+    ) -> Tuple[str, str]:
 
         addrs = [_addrs] if isinstance(_addrs, str) else _addrs
         hosts = [_hosts] if isinstance(_hosts, str) else _hosts
@@ -97,6 +102,8 @@ def generate_cert(self, _hosts: Union[str, List[str]], _addrs: Union[str, List[s
         san_list: List[x509.GeneralName] = [x509.DNSName(host) for host in hosts]
         if valid_ips:
             san_list.extend(ips)
+        if custom_san_list:
+            san_list.extend([x509.DNSName(n) for n in custom_san_list])
 
         builder = builder.add_extension(
             x509.SubjectAlternativeName(

src/python-common/ceph/deployment/service_spec.py

@@ -1313,6 +1313,10 @@ def validate(self) -> None:
             raise SpecValidationError('"ssl" field must be set to true when "generate_cert" '
                                       'is set to true')
 
+        if self.generate_cert and self.rgw_frontend_ssl_certificate:
+            raise SpecValidationError('"generate_cert" field and "rgw_frontend_ssl_certificate" '
+                                      'field are mutually exclusive')
+
 
 yaml.add_representer(RGWSpec, ServiceSpec.yaml_representer)