mgr/cephadm: Allow Ingress service to expose the metrics via HTTPS also add fields in spec to accept monitor ips/ monitor networks

ShwetaBhosale1 · ShwetaBhosale1 · commit da5d3ab0f32d · 2025-09-25T22:12:12.000+05:30
Fixes: https://tracker.ceph.com/issues/71707 Signed-off-by: Shweta Bhosale <Shweta.Bhosale1@ibm.com>
diff --git a/doc/cephadm/services/rgw.rst b/doc/cephadm/services/rgw.rst
@@ -409,13 +409,30 @@ Service specs are YAML blocks with the following properties:
       use_keepalived_multicast: <bool>          # optional: Default is False.
       vrrp_interface_network: <string>/<string> # optional: ex: 192.168.20.0/24
       health_check_interval: <string>           # optional: Default is 2s.
+      ssl: true
+      certificate_source: inline                # optional: Default is cephadm-signed
       ssl_cert: |                               # optional: SSL certificate and key
         -----BEGIN CERTIFICATE-----
         ...
         -----END CERTIFICATE-----
+      ssl_key: |
         -----BEGIN PRIVATE KEY-----
         ...
         -----END PRIVATE KEY-----
+      enable_stats: true
+      monitor_ssl: <bool>
+      monitor_cert_source: inline                       # optional: default is reuse_service_cert
+      monitor_ssl_cert: |                               # optional: SSL certificate and key
+        -----BEGIN CERTIFICATE-----
+        ...
+        -----END CERTIFICATE-----
+      monitor_ssl_key: |
+        -----BEGIN PRIVATE KEY-----
+        ...
+        -----END PRIVATE KEY-----
+      monitor_networks: [..]
+      monitor_ip_addrs:
+        host: <ip>
 
 .. code-block:: yaml
 
@@ -467,9 +484,20 @@ where the properties of this service specification are:
     A list of networks to identify which ethernet interface to use for the virtual IP.
 * ``frontend_port``
     The port used to access the ingress service.
-* ``ssl_cert``:
-    SSL certificate, if SSL is to be enabled. This must contain the both the certificate and
-    private key blocks in .pem format.
+* ``ssl``
+    To enable SSL for ingress service.
+* ``certificate_source``
+    The certificate source can be one of the following: 'inline', 'reference', or 'cephadm-signed'.
+    - If set to 'inline', the YAML configuration must include ssl_cert and ssl_key.
+    - If set to 'reference', the certificate and key must already exist in the certificate store.
+    - If set to 'cephadm-signed', Cephadm will automatically generate the certificate and key.
+    By default, the source is set to 'cephadm-signed'.
+* ``ssl_cert``
+    SSL certificate, if SSL is enabled and ``certificate_source`` is not 'cephadm-signed'.
+    This should have the certificate .pem format.
+* ``ssl_key``
+    SSL key, if SSL is enabled and ``certificate_source`` is not 'cephadm-signed'.
+    This should have the key .pem format.
 * ``use_keepalived_multicast``
     Default is False. By default, cephadm will deploy keepalived config to use unicast IPs,
     using the IPs of the hosts. The IPs chosen will be the same IPs cephadm uses to connect
@@ -488,6 +516,29 @@ where the properties of this service specification are:
 * ``health_check_interval``
     Default is 2 seconds. This parameter can be used to set the interval between health checks
     for the haproxy with the backend servers.
+* ``enable_stats``
+    Default is False, must be set to enable haproxy stats.
+* ``monitor_ssl``
+    To enable ssl for monitoring. SSL for monitoring can be enabled only when service SSL is enabled.
+* ``monitor_cert_source``
+    The monitor certificate source can be one of the following: 'reuse_service_cert', 'inline', 'reference', or 'cephadm-signed'.
+    - If set to 'reuse_service_cert', then the service certs will be used.
+    - If set to 'inline', the YAML configuration must include ssl_cert and ssl_key.
+    - If set to 'reference', the certificate and key must already exist in the certificate store.
+    - If set to 'cephadm-signed', Cephadm will automatically generate the certificate and key.
+    By default, the source is set to 'reuse_service_cert'.
+* ``monitor_ssl_cert``
+    Monitor SSL certificate, if monitor SSL is enabled and ``monitor_cert_source``
+    is not 'cephadm-signed'. This should have the certificate .pem format.
+* ``monitor_ssl_key``
+    Monitor SSL key, if monitor SSL is enabled and ``monitor_cert_source`` is not
+    'cephadm-signed'. This should have the key .pem format.
+* ``monitor_ip_addrs``
+    If ``monitor_ip_addrs`` is provided and the specified IP address is assigned to the host,
+    that IP address will be used. If IP address is not present, then 'monitor_networks' will be checked.
+* ``monitor_networks``
+    If ``monitor_networks`` is specified, an IP address that matches one of the specified
+    networks will be used. If IP not present, then default host ip will be used.
 
 .. _ingress-virtual-ip:
 
diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
@@ -739,6 +739,8 @@ def _init_cert_mgr(self) -> None:
 
         self.cert_mgr.register_cert_key_pair('nvmeof', 'nvmeof_client_cert', 'nvmeof_client_key', TLSObjectScope.SERVICE)
         self.cert_mgr.register_cert('nvmeof', 'nvmeof_root_ca_cert', TLSObjectScope.SERVICE)
+        # register haproxy monitor ssl cert and key
+        self.cert_mgr.register_cert_key_pair('ingress', 'haproxy_monitor_ssl_cert', 'haproxy_monitor_ssl_key', TLSObjectScope.SERVICE)
 
         self.cert_mgr.init_tlsobject_store()
 
diff --git a/src/pybind/mgr/cephadm/services/ingress.py b/src/pybind/mgr/cephadm/services/ingress.py
@@ -4,7 +4,7 @@
 import string
 from typing import List, Dict, Any, Tuple, cast, Optional, TYPE_CHECKING
 
-from ceph.deployment.service_spec import ServiceSpec, IngressSpec
+from ceph.deployment.service_spec import ServiceSpec, IngressSpec, MonitorCertSource
 from mgr_util import build_url
 from cephadm import utils
 from orchestrator import OrchestratorError, DaemonDescription
@@ -26,6 +26,14 @@ class IngressService(CephService):
     def needs_monitoring(self) -> bool:
         return True
 
+    @property
+    def haproxy_stats_cert_name(self) -> str:
+        return 'haproxy_monitor_ssl_cert'
+
+    @property
+    def haproxy_stats_key_name(self) -> str:
+        return 'haproxy_monitor_ssl_key'
+
     @classmethod
     def get_dependencies(cls, mgr: "CephadmOrchestrator",
                          spec: Optional[ServiceSpec] = None,
@@ -214,6 +222,23 @@ def haproxy_generate_config(
         frontend_port = daemon_spec.ports[0] if daemon_spec.ports else spec.frontend_port
         if ip != '[::]' and frontend_port:
             daemon_spec.port_ips = {str(frontend_port): ip}
+
+        monitor_ip, monitor_port = self.get_monitoring_details(daemon_spec.service_name, daemon_spec.host)
+        if monitor_ip:
+            monitor_ips = [monitor_ip]
+            daemon_spec.port_ips.update({str(monitor_port): monitor_ip})
+        else:
+            monitor_ips = [ip, host_ip]
+
+        monitor_ssl_file = None
+        cert_ips = [ip]
+        if spec.monitor_ssl:
+            if spec.monitor_cert_source == MonitorCertSource.REUSE_SERVICE_CERT.value:
+                monitor_ssl_file = 'haproxy.pem'
+                cert_ips.extend(monitor_ips)
+            else:
+                monitor_ssl_file = 'stats_haproxy.pem'
+
         haproxy_conf = self.mgr.template.render(
             'services/ingress/haproxy.cfg.j2',
             {
@@ -224,12 +249,13 @@ def haproxy_generate_config(
                 'user': spec.monitor_user or 'admin',
                 'password': password,
                 'ip': ip,
+                'monitor_ips': monitor_ips,
                 'frontend_port': frontend_port,
-                'monitor_port': daemon_spec.ports[1] if daemon_spec.ports else spec.monitor_port,
-                'local_host_ip': host_ip,
+                'monitor_port': spec.monitor_port,
                 'default_server_opts': server_opts,
                 'health_check_interval': spec.health_check_interval or '2s',
                 'v4v6_flag': v4v6_flag,
+                'monitor_ssl_file': monitor_ssl_file,
             }
         )
         config_files = {
@@ -243,8 +269,30 @@ def haproxy_generate_config(
             combined_pem = tls_pair.cert + '\n' + tls_pair.key
             config_files['files']['haproxy.pem'] = combined_pem
 
+        if spec.monitor_ssl and spec.monitor_cert_source != MonitorCertSource.REUSE_SERVICE_CERT.value:
+            stats_cert, stats_key = self.get_stats_certs(spec, daemon_spec, monitor_ips)
+            monitor_ssl_cert = [stats_cert, stats_key]
+            config_files['files']['stats_haproxy.pem'] = '\n'.join(monitor_ssl_cert)
+
         return config_files, self.get_haproxy_dependencies(self.mgr, spec)
 
+    def get_stats_certs(
+        self,
+        svc_spec: IngressSpec,
+        daemon_spec: CephadmDaemonDeploySpec,
+        ips: Optional[List[str]] = None,
+    ) -> Tuple[str, str]:
+        return self.get_certificates_generic(
+            svc_spec=svc_spec,
+            daemon_spec=daemon_spec,
+            cert_attr='monitor_ssl_cert',
+            key_attr='monitor_ssl_key',
+            cert_source_attr='monitor_cert_source',
+            cert_name=self.haproxy_stats_cert_name,
+            key_name=self.haproxy_stats_key_name,
+            ips=ips
+        )
+
     def keepalived_prepare_create(
             self,
             daemon_spec: CephadmDaemonDeploySpec,
@@ -445,3 +493,18 @@ def _get_valid_interface_and_ip(vip: str, host: str) -> Tuple[str, str]:
         }
 
         return config_file, self.get_keepalived_dependencies(self.mgr, spec)
+
+    def get_monitoring_details(self, service_name: str, host: str) -> Tuple[Optional[str], Optional[int]]:
+        spec = cast(IngressSpec, self.mgr.spec_store[service_name].spec)
+        monitor_port = spec.monitor_port
+
+        # check if monitor needs to be bind on specific ip
+        monitor_addr = spec.monitor_ip_addrs.get(host) if spec.monitor_ip_addrs else None
+        if monitor_addr and monitor_addr not in self.mgr.cache.get_host_network_ips(host):
+            logger.debug(f"Monitoring IP {monitor_addr} is not configured on host {host}.")
+            monitor_addr = None
+        if not monitor_addr and spec.monitor_networks:
+            monitor_addr = self.mgr.get_first_matching_network_ip(host, spec, spec.monitor_networks)
+            if not monitor_addr:
+                logger.debug(f"No IP address found in the network {spec.monitor_networks} on host {host}.")
+        return monitor_addr, monitor_port
diff --git a/src/pybind/mgr/cephadm/services/service_discovery.py b/src/pybind/mgr/cephadm/services/service_discovery.py
@@ -12,12 +12,12 @@ class Server:  # type: ignore
 from mgr_util import build_url
 from typing import Dict, List, TYPE_CHECKING, cast, Collection, Callable, NamedTuple, Optional, IO
 from cephadm.services.nfs import NFSService
+from cephadm.services.ingress import IngressService
 from cephadm.services.monitoring import AlertmanagerService, NodeExporterService, PrometheusService
 import secrets
 from mgr_util import verify_tls_files
 import tempfile
 
-from cephadm.services.ingress import IngressSpec
 from cephadm.services.cephadmservice import CephExporterService
 from cephadm.services.nvmeof import NvmeofService
 from cephadm.services.service_registry import service_registry
@@ -223,12 +223,13 @@ def haproxy_sd_config(self) -> List[Dict[str, Collection[str]]]:
         srv_entries = []
         for dd in self.mgr.cache.get_daemons_by_type('ingress'):
             if dd.service_name() in self.mgr.spec_store:
-                spec = cast(IngressSpec, self.mgr.spec_store[dd.service_name()].spec)
                 assert dd.hostname is not None
                 if dd.daemon_type == 'haproxy':
-                    addr = self.mgr.inventory.get_addr(dd.hostname)
+                    ingress = cast(IngressService, service_registry.get_service('ingress'))
+                    monitor_ip, monitor_port = ingress.get_monitoring_details(dd.service_name(), dd.hostname)
+                    addr = monitor_ip or dd.ip or self.mgr.inventory.get_addr(dd.hostname)
                     srv_entries.append({
-                        'targets': [f"{build_url(host=addr, port=spec.monitor_port).lstrip('/')}"],
+                        'targets': [f"{build_url(host=addr, port=monitor_port).lstrip('/')}"],
                         'labels': {'ingress': dd.service_name(), 'instance': dd.hostname}
                     })
         return srv_entries
diff --git a/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2 b/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2
@@ -45,17 +45,24 @@ defaults
 {% endif %}
     maxconn                 8000
 
+{% if spec.enable_stats %}
 frontend stats
     mode http
-    bind {{ ip }}:{{ monitor_port }}
-    bind {{ local_host_ip }}:{{ monitor_port }}
+{% for monitor_ip in monitor_ips %}
+    {% if spec.monitor_ssl %}
+    bind {{ monitor_ip }}:{{ monitor_port }} ssl crt /var/lib/haproxy/{{ monitor_ssl_file }}
+    {% else %}
+    bind {{ monitor_ip }}:{{ monitor_port }}
+    {% endif %}
+{% endfor %}
     stats enable
     stats uri /stats
     stats refresh 10s
     stats auth {{ user }}:{{ password }}
     http-request use-service prometheus-exporter if { path /metrics }
     monitor-uri /health
 
+{% endif %}
 frontend frontend
 {% if spec.ssl or spec.ssl_cert %}
     bind {{ ip }}:{{ frontend_port }} ssl crt /var/lib/haproxy/haproxy.pem {{ v4v6_flag }}
diff --git a/src/pybind/mgr/cephadm/tests/test_service_discovery.py b/src/pybind/mgr/cephadm/tests/test_service_discovery.py
@@ -68,6 +68,8 @@ def __init__(self, port):
 class FakeIngressServiceSpec:
     def __init__(self, port):
         self.monitor_port = port
+        self.monitor_ip_addrs = {}
+        self.monitor_networks = {}
 
 
 class FakeServiceSpec:
diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py
diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py
diff --git a/src/python-common/ceph/tests/test_service_spec.py b/src/python-common/ceph/tests/test_service_spec.py
