Merge pull request ceph#58815 from synarete/samba-metrics-exporter

cephadm: samba metrics exporter

Reviewed-by: Adam King <[email protected]>
Reviewed-by: Avan Thakkar <[email protected]>
Reviewed-by: John Mulligan <[email protected]>

Author: adk3798

src/cephadm/cephadmlib/constants.py

@@ -19,6 +19,7 @@
 DEFAULT_JAEGER_AGENT_IMAGE = 'quay.io/jaegertracing/jaeger-agent:1.29'
 DEFAULT_JAEGER_QUERY_IMAGE = 'quay.io/jaegertracing/jaeger-query:1.29'
 DEFAULT_SMB_IMAGE = 'quay.io/samba.org/samba-server:devbuilds-centos-amd64'
+DEFAULT_SMBMETRICS_IMAGE = 'quay.io/samba.org/samba-metrics:latest'
 DEFAULT_NGINX_IMAGE = 'quay.io/ceph/nginx:1.26.1'
 DEFAULT_OAUTH2_PROXY_IMAGE = 'quay.io/oauth2-proxy/oauth2-proxy:v7.6.0'
 DEFAULT_REGISTRY = 'docker.io'  # normalize unqualified digests to this

src/cephadm/cephadmlib/daemons/smb.py

@@ -81,6 +81,8 @@ class Config:
     smb_port: int
     ceph_config_entity: str
     vhostname: str
+    metrics_image: str
+    metrics_port: int
     # clustering related values
     rank: int
     rank_generation: int
@@ -103,6 +105,8 @@ def __init__(
         smb_port: int = 0,
         ceph_config_entity: str = 'client.admin',
         vhostname: str = '',
+        metrics_image: str = '',
+        metrics_port: int = 0,
         rank: int = -1,
         rank_generation: int = -1,
         cluster_meta_uri: str = '',
@@ -122,6 +126,8 @@ def __init__(
         self.smb_port = smb_port
         self.ceph_config_entity = ceph_config_entity
         self.vhostname = vhostname
+        self.metrics_image = metrics_image
+        self.metrics_port = metrics_port
         self.rank = rank
         self.rank_generation = rank_generation
         self.cluster_meta_uri = cluster_meta_uri
@@ -155,15 +161,34 @@ def _container_dns_args(cfg: Config) -> List[str]:
     return cargs
 
 
-class SambaContainerCommon:
-    def __init__(
-        self,
-        cfg: Config,
-    ) -> None:
+class ContainerCommon:
+    def __init__(self, cfg: Config, image: str = '') -> None:
         self.cfg = cfg
+        self.image = image
 
     def name(self) -> str:
-        raise NotImplementedError('samba container name')
+        raise NotImplementedError('container name')
+
+    def envs(self) -> Dict[str, str]:
+        return {}
+
+    def envs_list(self) -> List[str]:
+        return []
+
+    def args(self) -> List[str]:
+        return []
+
+    def container_args(self) -> List[str]:
+        return []
+
+    def container_image(self) -> str:
+        return self.image
+
+
+class SambaContainerCommon(ContainerCommon):
+    def __init__(self, cfg: Config, image: str = '') -> None:
+        self.cfg = cfg
+        self.image = image
 
     def envs(self) -> Dict[str, str]:
         environ = {
@@ -196,9 +221,6 @@ def args(self) -> List[str]:
             args.append(f'--debug-delay={self.cfg.debug_delay}')
         return args
 
-    def container_args(self) -> List[str]:
-        return []
-
 
 class SambaNetworkedInitContainer(SambaContainerCommon):
     """SambaContainerCommon subclass that enables additional networking
@@ -233,6 +255,9 @@ def container_args(self) -> List[str]:
         cargs = []
         if self.cfg.smb_port:
             cargs.append(f'--publish={self.cfg.smb_port}:{self.cfg.smb_port}')
+        if self.cfg.metrics_port:
+            metrics_port = self.cfg.metrics_port
+            cargs.append(f'--publish={metrics_port}:{metrics_port}')
         cargs.extend(_container_dns_args(self.cfg))
         return cargs
 
@@ -284,6 +309,17 @@ def args(self) -> List[str]:
         return super().args() + ['update-config', '--watch']
 
 
+class SMBMetricsContainer(ContainerCommon):
+    def name(self) -> str:
+        return 'smbmetrics'
+
+    def args(self) -> List[str]:
+        args = []
+        if self.cfg.metrics_port > 0:
+            args.append(f'--port={self.cfg.metrics_port}')
+        return args
+
+
 class CTDBMigrateInitContainer(SambaContainerCommon):
     def name(self) -> str:
         return 'ctdbMigrate'
@@ -358,13 +394,13 @@ def args(self) -> List[str]:
 class ContainerLayout:
     init_containers: List[SambaContainerCommon]
     primary: SambaContainerCommon
-    supplemental: List[SambaContainerCommon]
+    supplemental: List[ContainerCommon]
 
     def __init__(
         self,
         init_containers: List[SambaContainerCommon],
         primary: SambaContainerCommon,
-        supplemental: List[SambaContainerCommon],
+        supplemental: List[ContainerCommon],
     ) -> None:
         self.init_containers = init_containers
         self.primary = primary
@@ -393,6 +429,7 @@ def __init__(self, ctx: CephadmContext, ident: DaemonIdentity):
         self._cached_layout: Optional[ContainerLayout] = None
         self._rank_info = context_getters.fetch_rank_info(ctx)
         self.smb_port = 445
+        self.metrics_port = 9922
         self._network_mapper = _NetworkMapper(ctx)
         logger.debug('Created SMB ContainerDaemonForm instance')
 
@@ -431,6 +468,8 @@ def validate(self) -> None:
         files = data_utils.dict_get(configs, 'files', {})
         ceph_config_entity = configs.get('config_auth_entity', '')
         vhostname = configs.get('virtual_hostname', '')
+        metrics_image = configs.get('metrics_image', '')
+        metrics_port = int(configs.get('metrics_port', '0'))
         cluster_meta_uri = configs.get('cluster_meta_uri', '')
         cluster_lock_uri = configs.get('cluster_lock_uri', '')
         cluster_public_addrs = configs.get('cluster_public_addrs', [])
@@ -470,6 +509,8 @@ def validate(self) -> None:
             smb_port=self.smb_port,
             ceph_config_entity=ceph_config_entity,
             vhostname=vhostname,
+            metrics_image=metrics_image,
+            metrics_port=metrics_port,
             cluster_meta_uri=cluster_meta_uri,
             cluster_lock_uri=cluster_lock_uri,
             cluster_public_addrs=_public_addrs,
@@ -517,7 +558,7 @@ def _layout(self) -> ContainerLayout:
         if self._cached_layout:
             return self._cached_layout
         init_ctrs: List[SambaContainerCommon] = []
-        ctrs: List[SambaContainerCommon] = []
+        ctrs: List[ContainerCommon] = []
 
         init_ctrs.append(ConfigInitContainer(self._cfg))
         ctrs.append(ConfigWatchContainer(self._cfg))
@@ -526,6 +567,11 @@ def _layout(self) -> ContainerLayout:
             init_ctrs.append(MustJoinContainer(self._cfg))
             ctrs.append(WinbindContainer(self._cfg))
 
+        metrics_image = self._cfg.metrics_image.strip()
+        metrics_port = self._cfg.metrics_port
+        if metrics_image and metrics_port > 0:
+            ctrs.append(SMBMetricsContainer(self._cfg, metrics_image))
+
         if self._cfg.clustered:
             init_ctrs += [
                 CTDBMigrateInitContainer(self._cfg),
@@ -564,7 +610,7 @@ def _to_init_container(
         )
 
     def _to_sidecar_container(
-        self, ctx: CephadmContext, smb_ctr: SambaContainerCommon
+        self, ctx: CephadmContext, smb_ctr: ContainerCommon
     ) -> SidecarContainer:
         volume_mounts: Dict[str, str] = {}
         container_args: List[str] = smb_ctr.container_args()
@@ -587,10 +633,11 @@ def _to_sidecar_container(
         identity = DaemonSubIdentity.from_parent(
             self.identity, smb_ctr.name()
         )
+        img = smb_ctr.container_image() or ctx.image or self.default_image
         return SidecarContainer(
             ctx,
             entrypoint='',
-            image=ctx.image or self.default_image,
+            image=img,
             identity=identity,
             container_args=container_args,
             args=smb_ctr.args(),
@@ -673,6 +720,9 @@ def customize_container_endpoints(
     ) -> None:
         if not any(ep.port == self.smb_port for ep in endpoints):
             endpoints.append(EndPoint('0.0.0.0', self.smb_port))
+        if self.metrics_port > 0:
+            if not any(ep.port == self.metrics_port for ep in endpoints):
+                endpoints.append(EndPoint('0.0.0.0', self.metrics_port))
 
     def prepare_data_dir(self, data_dir: str, uid: int, gid: int) -> None:
         self.validate()

src/pybind/mgr/cephadm/module.py

@@ -148,6 +148,7 @@ def os_exit_noop(status: int) -> None:
 DEFAULT_OAUTH2_PROXY = 'quay.io/oauth2-proxy/oauth2-proxy:v7.6.0'
 DEFAULT_JAEGER_QUERY_IMAGE = 'quay.io/jaegertracing/jaeger-query:1.29'
 DEFAULT_SAMBA_IMAGE = 'quay.io/samba.org/samba-server:devbuilds-centos-amd64'
+DEFAULT_SAMBA_METRICS_IMAGE = 'quay.io/samba.org/samba-metrics:latest'
 # ------------------------------------------------------------------------------
 
 
@@ -319,6 +320,11 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule,
             default=DEFAULT_SAMBA_IMAGE,
             desc='Samba/SMB container image',
         ),
+        Option(
+            'container_image_samba_metrics',
+            default=DEFAULT_SAMBA_METRICS_IMAGE,
+            desc='Samba/SMB metrics exporter container image',
+        ),
         Option(
             'warn_on_stray_hosts',
             type='bool',
@@ -585,6 +591,7 @@ def __init__(self, *args: Any, **kwargs: Any):
             self.container_image_jaeger_collector = ''
             self.container_image_jaeger_query = ''
             self.container_image_samba = ''
+            self.container_image_samba_metrics = ''
             self.warn_on_stray_hosts = True
             self.warn_on_stray_daemons = True
             self.warn_on_failed_host_check = True

src/pybind/mgr/cephadm/service_discovery.py

@@ -13,6 +13,7 @@ class Server:  # type: ignore
 from mgr_util import build_url
 from typing import Dict, List, TYPE_CHECKING, cast, Collection, Callable, NamedTuple, Optional, IO
 from cephadm.services.nfs import NFSService
+from cephadm.services.smb import SMBService
 from cephadm.services.monitoring import AlertmanagerService, NodeExporterService, PrometheusService
 import secrets
 from mgr_util import verify_tls_files
@@ -149,6 +150,7 @@ def index(self) -> str:
 <p><a href='prometheus/sd-config?service=ceph-exporter'>Ceph exporter http sd-config</a></p>
 <p><a href='prometheus/sd-config?service=nvmeof'>NVMeoF http sd-config</a></p>
 <p><a href='prometheus/sd-config?service=nfs'>NFS http sd-config</a></p>
+<p><a href='prometheus/sd-config?service=smb'>SMB http sd-config</a></p>
 <p><a href='prometheus/rules'>Prometheus rules</a></p>
 </body>
 </html>'''
@@ -171,6 +173,8 @@ def get_sd_config(self, service: str) -> List[Dict[str, Collection[str]]]:
             return self.nvmeof_sd_config()
         elif service == 'nfs':
             return self.nfs_sd_config()
+        elif service == 'smb':
+            return self.smb_sd_config()
         else:
             return []
 
@@ -265,6 +269,19 @@ def nfs_sd_config(self) -> List[Dict[str, Collection[str]]]:
             })
         return srv_entries
 
+    def smb_sd_config(self) -> List[Dict[str, Collection[str]]]:
+        """Return <http_sd_config> compatible prometheus config for smb service."""
+        srv_entries = []
+        for dd in self.mgr.cache.get_daemons_by_type('smb'):
+            assert dd.hostname is not None
+            addr = dd.ip if dd.ip else self.mgr.inventory.get_addr(dd.hostname)
+            port = SMBService.DEFAULT_EXPORTER_PORT
+            srv_entries.append({
+                'targets': [build_url(host=addr, port=port).lstrip('/')],
+                'labels': {'instance': dd.hostname}
+            })
+        return srv_entries
+
     @cherrypy.expose(alias='prometheus/rules')
     def get_prometheus_rules(self) -> str:
         """Return currently configured prometheus rules as Yaml."""

src/pybind/mgr/cephadm/services/monitoring.py

@@ -503,6 +503,7 @@ def generate_config(
         nvmeof_sd_url = f'{srv_end_point}service=nvmeof'  # always included
         mgmt_gw_enabled = len(self.mgr.cache.get_daemons_by_service('mgmt-gateway')) > 0
         nfs_sd_url = f'{srv_end_point}service=nfs'  # always included
+        smb_sd_url = f'{srv_end_point}service=smb'  # always included
 
         alertmanager_user, alertmanager_password = self.mgr._get_alertmanager_credentials()
         prometheus_user, prometheus_password = self.mgr._get_prometheus_credentials()
@@ -524,7 +525,8 @@ def generate_config(
             'nvmeof_sd_url': nvmeof_sd_url,
             'external_prometheus_targets': targets,
             'cluster_fsid': FSID,
-            'nfs_sd_url': nfs_sd_url
+            'nfs_sd_url': nfs_sd_url,
+            'smb_sd_url': smb_sd_url
         }
 
         ip_to_bind_to = ''

src/pybind/mgr/cephadm/services/smb.py

@@ -16,6 +16,7 @@
 
 class SMBService(CephService):
     TYPE = 'smb'
+    DEFAULT_EXPORTER_PORT = 9922
     smb_pool = '.smb'  # minor layering violation. try to clean up later.
 
     def config(self, spec: ServiceSpec) -> None:
@@ -79,6 +80,11 @@ def generate_config(
                 smb_spec, daemon_spec.daemon_id, ceph_users
             )
         )
+        config_blobs['metrics_image'] = (
+            self.mgr.container_image_samba_metrics
+        )
+        config_blobs['metrics_port'] = SMBService.DEFAULT_EXPORTER_PORT
+
         logger.debug('smb generate_config: %r', config_blobs)
         self._configure_cluster_meta(smb_spec, daemon_spec)
         return config_blobs, []

src/pybind/mgr/cephadm/templates/services/prometheus/prometheus.yml.j2

@@ -181,6 +181,26 @@ scrape_configs:
 {% endif %}
 {% endif %}
 
+{% if smb_sd_url %}
+  - job_name: 'smb'
+{% if security_enabled %}
+    honor_labels: true
+    scheme: https
+    tls_config:
+      ca_file: root_cert.pem
+    http_sd_configs:
+    - url: {{ smb_sd_url }}
+      basic_auth:
+        username: {{ service_discovery_username }}
+        password: {{ service_discovery_password }}
+      tls_config:
+        ca_file: root_cert.pem
+{% else %}
+    http_sd_configs:
+    - url: {{ smb_sd_url }}
+{% endif %}
+{% endif %}
+
 {% if not security_enabled %}
   - job_name: 'federate'
     scrape_interval: 15s

src/pybind/mgr/cephadm/tests/test_service_discovery.py

@@ -27,6 +27,10 @@ def get_daemons_by_service(self, service_type):
             return [FakeDaemonDescription('1.2.3.4', [9587], 'node0'),
                     FakeDaemonDescription('1.2.3.5', [9587], 'node1')]
 
+        if service_type == 'smb':
+            return [FakeDaemonDescription('1.2.3.4', [9922], 'node0'),
+                    FakeDaemonDescription('1.2.3.5', [9922], 'node1')]
+
         return [FakeDaemonDescription('1.2.3.4', [9100], 'node0'),
                 FakeDaemonDescription('1.2.3.5', [9200], 'node1')]
 
@@ -206,6 +210,20 @@ def test_get_sd_config_nfs(self):
         # check content
         assert cfg[0]['targets'] == ['1.2.3.4:9587']
 
+    def test_get_sd_config_smb(self):
+        mgr = FakeMgr()
+        root = Root(mgr, 5000, '0.0.0.0')
+        cfg = root.get_sd_config('smb')
+
+        # check response structure
+        assert cfg
+        for entry in cfg:
+            assert 'labels' in entry
+            assert 'targets' in entry
+
+        # check content
+        assert cfg[0]['targets'] == ['1.2.3.4:9922']
+
     def test_get_sd_config_invalid_service(self):
         mgr = FakeMgr()
         root = Root(mgr, 5000, '0.0.0.0')