mgr/cephadm: adding new cephadm service mgmt-gateway

adding mgmt-gateway, a new cephadm service based on nginx, to act as
the front-end and single entry point to the cluster. This gateway
offers unified access to all Ceph applications, including the
Ceph dashboard and monitoring tools (Prometheus, Grafana, ..),
while enhancing security and simplifying access management
through nginx.

Fixes: https://tracker.ceph.com/issues/66095

Signed-off-by: Redouane Kachach <[email protected]>

Author: rkachach

src/cephadm/cephadm.py

@@ -176,6 +176,7 @@
     NFSGanesha,
     SMB,
     SNMPGateway,
+    MgmtGateway,
     Tracing,
     NodeProxy,
 )
@@ -227,6 +228,7 @@ def get_supported_daemons():
     supported_daemons.append(Keepalived.daemon_type)
     supported_daemons.append(CephadmAgent.daemon_type)
     supported_daemons.append(SNMPGateway.daemon_type)
+    supported_daemons.append(MgmtGateway.daemon_type)
     supported_daemons.extend(Tracing.components)
     supported_daemons.append(NodeProxy.daemon_type)
     supported_daemons.append(SMB.daemon_type)
@@ -463,6 +465,8 @@ def update_default_image(ctx: CephadmContext) -> None:
             ctx.image = Keepalived.default_image
         if type_ == SNMPGateway.daemon_type:
             ctx.image = SNMPGateway.default_image
+        if type_ == MgmtGateway.daemon_type:
+            ctx.image = MgmtGateway.default_image
         if type_ == CephNvmeof.daemon_type:
             ctx.image = CephNvmeof.default_image
         if type_ in Tracing.components:
@@ -855,6 +859,10 @@ def create_daemon_dirs(
         sg = SNMPGateway.init(ctx, fsid, ident.daemon_id)
         sg.create_daemon_conf()
 
+    elif daemon_type == MgmtGateway.daemon_type:
+        cg = MgmtGateway.init(ctx, fsid, ident.daemon_id)
+        cg.create_daemon_dirs(data_dir, uid, gid)
+
     elif daemon_type == NodeProxy.daemon_type:
         node_proxy = NodeProxy.init(ctx, fsid, ident.daemon_id)
         node_proxy.create_daemon_dirs(data_dir, uid, gid)
@@ -3571,6 +3579,9 @@ def list_daemons(
                                 elif daemon_type == SNMPGateway.daemon_type:
                                     version = SNMPGateway.get_version(ctx, fsid, daemon_id)
                                     seen_versions[image_id] = version
+                                elif daemon_type == MgmtGateway.daemon_type:
+                                    version = MgmtGateway.get_version(ctx, container_id)
+                                    seen_versions[image_id] = version
                                 else:
                                     logger.warning('version for unknown daemon type %s' % daemon_type)
                         else:

src/cephadm/cephadmlib/constants.py

@@ -19,6 +19,7 @@
 DEFAULT_JAEGER_AGENT_IMAGE = 'quay.io/jaegertracing/jaeger-agent:1.29'
 DEFAULT_JAEGER_QUERY_IMAGE = 'quay.io/jaegertracing/jaeger-query:1.29'
 DEFAULT_SMB_IMAGE = 'quay.io/samba.org/samba-server:devbuilds-centos-amd64'
+DEFAULT_NGINX_IMAGE = 'quay.io/ceph/nginx:1.26.1'
 DEFAULT_REGISTRY = 'docker.io'  # normalize unqualified digests to this
 # ------------------------------------------------------------------------------
 

src/cephadm/cephadmlib/daemons/__init__.py

@@ -9,6 +9,7 @@
 from .snmp import SNMPGateway
 from .tracing import Tracing
 from .node_proxy import NodeProxy
+from .mgmt_gateway import MgmtGateway
 
 __all__ = [
     'Ceph',
@@ -25,4 +26,5 @@
     'SNMPGateway',
     'Tracing',
     'NodeProxy',
+    'MgmtGateway',
 ]

src/cephadm/cephadmlib/daemons/mgmt_gateway.py

@@ -0,0 +1,174 @@
+import logging
+import os
+from typing import Dict, List, Tuple, Optional
+import re
+
+from ..call_wrappers import call, CallVerbosity
+from ..container_daemon_form import ContainerDaemonForm, daemon_to_container
+from ..container_types import CephContainer
+from ..context import CephadmContext
+from ..context_getters import fetch_configs
+from ..daemon_form import register as register_daemon_form
+from ..daemon_identity import DaemonIdentity
+from ..deployment_utils import to_deployment_container
+from ..constants import DEFAULT_NGINX_IMAGE
+from ..data_utils import dict_get, is_fsid
+from ..file_utils import populate_files, makedirs, recursive_chown
+from ..exceptions import Error
+
+logger = logging.getLogger()
+
+
+@register_daemon_form
+class MgmtGateway(ContainerDaemonForm):
+    """Defines an MgmtGateway container"""
+
+    daemon_type = 'mgmt-gateway'
+    required_files = [
+        'nginx.conf',
+        'nginx_external_server.conf',
+        'nginx_internal_server.conf',
+        'nginx_internal.crt',
+        'nginx_internal.key',
+    ]
+
+    default_image = DEFAULT_NGINX_IMAGE
+
+    @classmethod
+    def for_daemon_type(cls, daemon_type: str) -> bool:
+        return cls.daemon_type == daemon_type
+
+    def __init__(
+        self,
+        ctx: CephadmContext,
+        fsid: str,
+        daemon_id: str,
+        config_json: Dict,
+        image: str = DEFAULT_NGINX_IMAGE,
+    ):
+        self.ctx = ctx
+        self.fsid = fsid
+        self.daemon_id = daemon_id
+        self.image = image
+        self.files = dict_get(config_json, 'files', {})
+        self.validate()
+
+    @classmethod
+    def init(
+        cls, ctx: CephadmContext, fsid: str, daemon_id: str
+    ) -> 'MgmtGateway':
+        return cls(ctx, fsid, daemon_id, fetch_configs(ctx), ctx.image)
+
+    @classmethod
+    def create(
+        cls, ctx: CephadmContext, ident: DaemonIdentity
+    ) -> 'MgmtGateway':
+        return cls.init(ctx, ident.fsid, ident.daemon_id)
+
+    @property
+    def identity(self) -> DaemonIdentity:
+        return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
+
+    def validate(self) -> None:
+        if not is_fsid(self.fsid):
+            raise Error(f'not an fsid: {self.fsid}')
+        if not self.daemon_id:
+            raise Error(f'invalid daemon_id: {self.daemon_id}')
+        if not self.image:
+            raise Error(f'invalid image: {self.image}')
+
+        # check for the required files
+        if self.required_files:
+            for fname in self.required_files:
+                if fname not in self.files:
+                    raise Error(
+                        'required file missing from config-json: %s' % fname
+                    )
+
+    def container(self, ctx: CephadmContext) -> CephContainer:
+        ctr = daemon_to_container(ctx, self)
+        return to_deployment_container(ctx, ctr)
+
+    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
+        return 65534, 65534  # nobody, nobody
+
+    def get_daemon_args(self) -> List[str]:
+        return []
+
+    def default_entrypoint(self) -> str:
+        return ''
+
+    def create_daemon_dirs(self, data_dir: str, uid: int, gid: int) -> None:
+        """Create files under the container data dir"""
+        if not os.path.isdir(data_dir):
+            raise OSError('data_dir is not a directory: %s' % (data_dir))
+        logger.info('Writing mgmt-gateway config...')
+        config_dir = os.path.join(data_dir, 'etc/')
+        makedirs(config_dir, uid, gid, 0o755)
+        recursive_chown(config_dir, uid, gid)
+        populate_files(config_dir, self.files, uid, gid)
+
+    def _get_container_mounts(self, data_dir: str) -> Dict[str, str]:
+        mounts: Dict[str, str] = {}
+        mounts[
+            os.path.join(data_dir, 'nginx.conf')
+        ] = '/etc/nginx/nginx.conf:Z'
+        return mounts
+
+    @staticmethod
+    def get_version(ctx: CephadmContext, container_id: str) -> Optional[str]:
+        """Return the version of the Nginx container"""
+        version = None
+        out, err, code = call(
+            ctx,
+            [
+                ctx.container_engine.path,
+                'exec',
+                container_id,
+                'nginx',
+                '-v',
+            ],
+            verbosity=CallVerbosity.QUIET,
+        )
+        if code == 0:
+            # nginx is using stderr to print the version!!
+            match = re.search(r'nginx version:\s*nginx\/(.+)', err)
+            if match:
+                version = match.group(1)
+        return version
+
+    def customize_container_mounts(
+        self, ctx: CephadmContext, mounts: Dict[str, str]
+    ) -> None:
+        data_dir = self.identity.data_dir(ctx.data_dir)
+        mounts.update(
+            {
+                os.path.join(
+                    data_dir, 'etc/nginx.conf'
+                ): '/etc/nginx/nginx.conf:Z',
+                os.path.join(
+                    data_dir, 'etc/nginx_internal_server.conf'
+                ): '/etc/nginx_internal_server.conf:Z',
+                os.path.join(
+                    data_dir, 'etc/nginx_external_server.conf'
+                ): '/etc/nginx_external_server.conf:Z',
+                os.path.join(
+                    data_dir, 'etc/nginx_internal.crt'
+                ): '/etc/nginx/ssl/nginx_internal.crt:Z',
+                os.path.join(
+                    data_dir, 'etc/nginx_internal.key'
+                ): '/etc/nginx/ssl/nginx_internal.key:Z',
+            }
+        )
+
+        if 'nginx.crt' in self.files:
+            mounts.update(
+                {
+                    os.path.join(
+                        data_dir, 'etc/nginx.crt'
+                    ): '/etc/nginx/ssl/nginx.crt:Z',
+                    os.path.join(
+                        data_dir, 'etc/nginx.key'
+                    ): '/etc/nginx/ssl/nginx.key:Z',
+                }
+            )

src/cephadm/cephadmlib/daemons/monitoring.py

@@ -260,6 +260,7 @@ def get_daemon_args(self) -> List[str]:
                 retention_size = config.get(
                     'retention_size', '0'
                 )  # default to disabled
+                use_url_prefix = config.get('use_url_prefix', False)
                 r += [f'--storage.tsdb.retention.time={retention_time}']
                 r += [f'--storage.tsdb.retention.size={retention_size}']
                 scheme = 'http'
@@ -271,10 +272,17 @@ def get_daemon_args(self) -> List[str]:
                     # use the first ipv4 (if any) otherwise use the first ipv6
                     addr = next(iter(ipv4_addrs or ipv6_addrs), None)
                     host = wrap_ipv6(addr) if addr else host
-                r += [f'--web.external-url={scheme}://{host}:{port}']
+                if use_url_prefix:
+                    r += [
+                        f'--web.external-url={scheme}://{host}:{port}/prometheus'
+                    ]
+                    r += ['--web.route-prefix=/prometheus/']
+                else:
+                    r += [f'--web.external-url={scheme}://{host}:{port}']
             r += [f'--web.listen-address={ip}:{port}']
         if daemon_type == 'alertmanager':
             config = fetch_configs(ctx)
+            use_url_prefix = config.get('use_url_prefix', False)
             peers = config.get('peers', list())  # type: ignore
             for peer in peers:
                 r += ['--cluster.peer={}'.format(peer)]
@@ -284,6 +292,8 @@ def get_daemon_args(self) -> List[str]:
                 pass
             # some alertmanager, by default, look elsewhere for a config
             r += ['--config.file=/etc/alertmanager/alertmanager.yml']
+            if use_url_prefix:
+                r += ['--web.route-prefix=/alertmanager']
         if daemon_type == 'promtail':
             r += ['--config.expand-env']
         if daemon_type == 'prometheus':

src/pybind/mgr/cephadm/inventory.py

@@ -1889,6 +1889,7 @@ def _init_known_cert_key_dicts(self) -> None:
             'iscsi_ssl_cert': {},  # service-name -> cert
             'ingress_ssl_cert': {},  # service-name -> cert
             'agent_endpoint_root_cert': Cert(),  # cert
+            'mgmt_gw_root_cert': Cert(),  # cert
             'service_discovery_root_cert': Cert(),  # cert
             'grafana_cert': {},  # host -> cert
             'alertmanager_cert': {},  # host -> cert
@@ -1901,6 +1902,7 @@ def _init_known_cert_key_dicts(self) -> None:
         self.known_keys = {
             'agent_endpoint_key': PrivKey(),  # key
             'service_discovery_key': PrivKey(),  # key
+            'mgmt_gw_root_key': PrivKey(),  # cert
             'grafana_key': {},  # host -> key
             'alertmanager_key': {},  # host -> key
             'prometheus_key': {},  # host -> key