Merge pull request ceph#64763 from adk3798/cephadm-limit-list-servers-calls

mgr/cephadm: limit calls to list_servers

Reviewed-by: John Mulligan <[email protected]>

Author: adk3798

src/pybind/mgr/cephadm/module.py

@@ -204,6 +204,12 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule,
             default=10 * 60,
             desc='how frequently to perform a host check',
         ),
+        Option(
+            'stray_daemon_check_interval',
+            type='secs',
+            default=30 * 60,
+            desc='how frequently cephadm should check for the presence of stray daemons',
+        ),
         Option(
             'mode',
             type='str',
@@ -519,6 +525,7 @@ def __init__(self, *args: Any, **kwargs: Any):
             self.daemon_cache_timeout = 0
             self.facts_cache_timeout = 0
             self.host_check_interval = 0
+            self.stray_daemon_check_interval = 0
             self.max_count_per_host = 0
             self.mode = ''
             self.container_image_base = ''
@@ -693,6 +700,8 @@ def __init__(self, *args: Any, **kwargs: Any):
 
         self.ceph_volume: CephVolume = CephVolume(self)
 
+        self.last_stray_daemon_check: Optional[datetime.datetime] = None
+
     def shutdown(self) -> None:
         self.log.debug('shutdown')
         self._worker_pool.close()

src/pybind/mgr/cephadm/serve.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+import datetime
 import ipaddress
 import hashlib
 import json
@@ -64,7 +64,7 @@ class CephadmServe:
     def __init__(self, mgr: "CephadmOrchestrator"):
         self.mgr: "CephadmOrchestrator" = mgr
         self.log = logger
-        self.last_certificates_check: Optional[datetime] = None
+        self.last_certificates_check: Optional[datetime.datetime] = None
 
     def serve(self) -> None:
         """
@@ -173,6 +173,7 @@ def _serve_sleep(self) -> None:
                 self.mgr.facts_cache_timeout,
                 self.mgr.daemon_cache_timeout,
                 self.mgr.device_cache_timeout,
+                self.mgr.stray_daemon_check_interval,
             )
         )
         self.log.debug('Sleeping for %d seconds', sleep_interval)
@@ -466,6 +467,9 @@ def _run_async_actions(self) -> None:
             (self.mgr.scheduled_async_actions.pop(0))()
 
     def _check_for_strays(self) -> None:
+        cutoff = datetime_now() - datetime.timedelta(seconds=self.mgr.stray_daemon_check_interval)
+        if self.mgr.last_stray_daemon_check is not None and self.mgr.last_stray_daemon_check >= cutoff:
+            return
         self.log.debug('_check_for_strays')
         for k in ['CEPHADM_STRAY_HOST',
                   'CEPHADM_STRAY_DAEMON']:
@@ -516,6 +520,7 @@ def _check_for_strays(self) -> None:
             if self.mgr.warn_on_stray_daemons and daemon_detail:
                 self.mgr.set_health_warning(
                     'CEPHADM_STRAY_DAEMON', f'{len(daemon_detail)} stray daemon(s) not managed by cephadm', len(daemon_detail), daemon_detail)
+            self.mgr.last_stray_daemon_check = datetime_now()
 
     def _service_reference_name(self, service_type: str, daemon_id: str) -> str:
         if service_type not in ['rbd-mirror', 'cephfs-mirror', 'rgw', 'rgw-nfs']:

src/pybind/mgr/cephadm/service_discovery.py

@@ -9,7 +9,6 @@ class Server:  # type: ignore
 import logging
 
 import orchestrator  # noqa
-from mgr_module import ServiceInfoT
 from mgr_util import build_url
 from typing import Dict, List, TYPE_CHECKING, cast, Collection, Callable, NamedTuple, Optional, IO
 from cephadm.services.nfs import NFSService
@@ -184,17 +183,16 @@ def get_sd_config(self, service: str) -> List[Dict[str, Collection[str]]]:
             return []
 
     def prometheus_sd_config(self) -> List[Dict[str, Collection[str]]]:
-        """Return <http_sd_config> compatible prometheus config for prometheus service."""
-        servers = self.mgr.list_servers()
+        """Return <http_sd_config> compatible prometheus config for prometheus service.
+        Targets should be a length one list containing only the active mgr
+        """
         targets = []
-        for server in servers:
-            hostname = server.get('hostname', '')
-            for service in cast(List[ServiceInfoT], server.get('services', [])):
-                if service['type'] != 'mgr' or service['id'] != self.mgr.get_mgr_id():
-                    continue
-                port = self.mgr.get_module_option_ex(
-                    'prometheus', 'server_port', PrometheusService.DEFAULT_MGR_PROMETHEUS_PORT)
-                targets.append(f'{hostname}:{port}')
+        mgr_daemons = self.mgr.cache.get_daemons_by_service('mgr')
+        host = service_registry.get_service('mgr').get_active_daemon(mgr_daemons).hostname or ''
+        fqdn = self.mgr.get_fqdn(host)
+        port = self.mgr.get_module_option_ex(
+            'prometheus', 'server_port', PrometheusService.DEFAULT_MGR_PROMETHEUS_PORT)
+        targets.append(f'{fqdn}:{port}')
         return [{"targets": targets, "labels": {}}]
 
     def alertmgr_sd_config(self) -> List[Dict[str, Collection[str]]]:

src/pybind/mgr/cephadm/tests/test_service_discovery.py

@@ -4,12 +4,13 @@
 
 
 class FakeDaemonDescription:
-    def __init__(self, ip, ports, hostname, service_name='', daemon_type=''):
+    def __init__(self, ip, ports, hostname, service_name='', daemon_type='', daemon_id=''):
         self.ip = ip
         self.ports = ports
         self.hostname = hostname
         self._service_name = service_name
         self.daemon_type = daemon_type
+        self.daemon_id = daemon_id if daemon_id else hostname
 
     def service_name(self):
         return self._service_name
@@ -36,6 +37,10 @@ def get_daemons_by_service(self, service_type):
             return [FakeDaemonDescription('1.2.3.4', [9123], 'node0'),
                     FakeDaemonDescription('1.2.3.5', [9123], 'node1')]
 
+        if service_type == 'mgr':
+            return [FakeDaemonDescription('1.2.3.4', [9922], 'node0', daemon_type='mgr', daemon_id='fake_active_mgr'),
+                    FakeDaemonDescription('1.2.3.5', [9922], 'node1', daemon_type='mgr', daemon_id='fake_standby_mgr')]
+
         return [FakeDaemonDescription('1.2.3.4', [9100], 'node0'),
                 FakeDaemonDescription('1.2.3.5', [9200], 'node1')]
 
@@ -111,19 +116,6 @@ def __init__(self):
     def get_mgr_id(self):
         return 'mgr-1'
 
-    def list_servers(self):
-
-        servers = [
-            {'hostname': 'node0',
-             'ceph_version': '16.2',
-             'services': [{'type': 'mgr', 'id': 'mgr-1'}, {'type': 'mon'}]},
-            {'hostname': 'node1',
-             'ceph_version': '16.2',
-             'services': [{'type': 'mgr', 'id': 'mgr-2'}, {'type': 'mon'}]}
-        ]
-
-        return servers
-
     def _check_mon_command(self, cmd_dict, inbuf=None):
         prefix = cmd_dict.get('prefix')
         if prefix == 'get-cmd':
@@ -136,6 +128,14 @@ def _check_mon_command(self, cmd_dict, inbuf=None):
     def get_module_option_ex(self, module, option, default_value):
         return "9283"
 
+    def daemon_is_self(self, d_type, d_id) -> bool:
+        if d_type == 'mgr' and d_id == 'fake_active_mgr':
+            return True
+        return False
+
+    def get_fqdn(self, hostname: str) -> str:
+        return hostname
+
 
 class TestServiceDiscovery: