mgr/cephadm: introduce deployment ordering for services

Some services don't work well with deploying every
daemon in parallel and need deployment to follow
a certain ordering. This commit sets up the framework
for allowing services to specify the order in which
its daemons should be deployed. The expectation is
that the function will return a dict mapping integers
to a list of daemons. When we deploy, we first deploy
everything in the "0" entry, then "1" etc. The integers
themselves are fairly arbitrary, we just need something
sortable.

Signed-off-by: Adam King <[email protected]>
(cherry picked from commit e8eb36d)

Resolves: rhbz#2372821

Author: adk3798

src/pybind/mgr/cephadm/serve.py

@@ -642,7 +642,7 @@ def _apply_all_services(self) -> bool:
         hosts_altered: Set[str] = set()
         all_conflicting_daemons: List[orchestrator.DaemonDescription] = []
         all_daemons_needing_fencing: List[orchestrator.DaemonDescription] = []
-        all_daemons_to_deploy: List[CephadmDaemonDeploySpec] = []
+        all_daemons_to_deploy: Dict[int, List[CephadmDaemonDeploySpec]] = {}
         all_daemons_to_remove: List[orchestrator.DaemonDescription] = []
         services_in_need_of_fencing: List[Tuple[ServiceSpec, Dict[int, Dict[int, Optional[str]]]]] = []
         for spec in specs:
@@ -654,7 +654,12 @@ def _apply_all_services(self) -> bool:
                 # compiles all these lists so we can split on host instead of service
                 all_conflicting_daemons.extend(conflicting_daemons)
                 all_daemons_needing_fencing.extend(daemons_to_fence)
-                all_daemons_to_deploy.extend(daemons_to_deploy)
+                svc = service_registry.get_service(spec.service_type)
+                service_deploy_ordering = svc.get_daemon_deployment_ordering(daemons_to_deploy)
+                for tier, daemons_of_tier in service_deploy_ordering.items():
+                    if tier not in all_daemons_to_deploy:
+                        all_daemons_to_deploy[tier] = []
+                    all_daemons_to_deploy[tier].extend(daemons_of_tier)
                 all_daemons_to_remove.extend(daemons_to_remove)
                 if service_needs_fencing and rank_map is not None:
                     services_in_need_of_fencing.append((spec, rank_map))
@@ -721,14 +726,20 @@ async def _deploy_and_remove_all(
 
             return await gather(*futures)
 
-        deploy_names = [d.name() for d in all_daemons_to_deploy]
         rm_names = (
             [d.name() for d in all_daemons_needing_fencing]
             + [d.name() for d in all_conflicting_daemons]
             + [d.name() for d in all_daemons_to_remove]
         )
-        with self.mgr.async_timeout_handler(cmd=f'cephadm deploying ({deploy_names} and removing {rm_names} daemons)'):
-            results = self.mgr.wait_async(_deploy_and_remove_all(all_daemons_needing_fencing, all_conflicting_daemons, all_daemons_to_deploy, all_daemons_to_remove))
+        for tier in sorted(all_daemons_to_deploy.keys()):
+            all_daemons_in_tier_to_deploy = all_daemons_to_deploy[tier]
+            deploy_names = [d.name() for d in all_daemons_in_tier_to_deploy]
+            if tier == 0:
+                with self.mgr.async_timeout_handler(cmd=f'cephadm deploying ({deploy_names} and removing {rm_names} daemons)'):
+                    results = self.mgr.wait_async(_deploy_and_remove_all(all_daemons_needing_fencing, all_conflicting_daemons, all_daemons_in_tier_to_deploy, all_daemons_to_remove))
+            else:
+                with self.mgr.async_timeout_handler(cmd=f'cephadm deploying ({deploy_names} daemons)'):
+                    results = self.mgr.wait_async(_deploy_and_remove_all([], [], all_daemons_in_tier_to_deploy, []))
 
         if any(res[0] for res in results):
             changed = True

src/pybind/mgr/cephadm/services/cephadmservice.py

@@ -359,6 +359,14 @@ def config(self, spec: ServiceSpec) -> None:
         """
         pass
 
+    def get_daemon_deployment_ordering(self, daemons: List[CephadmDaemonDeploySpec]) -> Dict[int, List[CephadmDaemonDeploySpec]]:
+        # for services that need to deploy their daemons in a certain order
+        # this will return a dict of ints (the integers are arbitrary and just count up)
+        # to a list of daemons it is okay to deploy in parallel. The expectation being
+        # we'll deploy all the daemons in the "0" entry, then all in "1" etc.
+        # The default behavior will just be to put all daemons in the "0" entry
+        return {0: daemons}
+
     def daemon_check_post(self, daemon_descrs: List[DaemonDescription]) -> None:
         """The post actions needed to be done after daemons are checked"""
         if self.mgr.config_dashboard: