Add Modular Manager Selector Interface (#3547)

Added a Manager Selector interface which allows users to choose an algorithm to sort the interesting managers. This will allow for flexible testing and implementation of manager selection strategies to optimize efficiency of the interchange.

Author: matthewc2003

parsl/executors/high_throughput/executor.py

@@ -20,6 +20,10 @@
 from parsl.executors.errors import BadMessage, ScalingFailed
 from parsl.executors.high_throughput import zmq_pipes
 from parsl.executors.high_throughput.errors import CommandClientTimeoutError
+from parsl.executors.high_throughput.manager_selector import (
+    ManagerSelector,
+    RandomManagerSelector,
+)
 from parsl.executors.high_throughput.mpi_prefix_composer import (
     VALID_LAUNCHERS,
     validate_resource_spec,
@@ -261,6 +265,7 @@ def __init__(self,
                  worker_logdir_root: Optional[str] = None,
                  enable_mpi_mode: bool = False,
                  mpi_launcher: str = "mpiexec",
+                 manager_selector: ManagerSelector = RandomManagerSelector(),
                  block_error_handler: Union[bool, Callable[[BlockProviderExecutor, Dict[str, JobStatus]], None]] = True,
                  encrypted: bool = False):
 
@@ -276,6 +281,7 @@ def __init__(self,
         self.prefetch_capacity = prefetch_capacity
         self.address = address
         self.address_probe_timeout = address_probe_timeout
+        self.manager_selector = manager_selector
         if self.address:
             self.all_addresses = address
         else:
@@ -544,6 +550,7 @@ def _start_local_interchange_process(self) -> None:
                               "poll_period": self.poll_period,
                               "logging_level": logging.DEBUG if self.worker_debug else logging.INFO,
                               "cert_dir": self.cert_dir,
+                              "manager_selector": self.manager_selector,
                               }
 
         config_pickle = pickle.dumps(interchange_config)

parsl/executors/high_throughput/interchange.py

@@ -6,7 +6,6 @@
 import pickle
 import platform
 import queue
-import random
 import signal
 import sys
 import threading
@@ -19,6 +18,7 @@
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.executors.high_throughput.errors import ManagerLost, VersionMismatch
 from parsl.executors.high_throughput.manager_record import ManagerRecord
+from parsl.executors.high_throughput.manager_selector import ManagerSelector
 from parsl.monitoring.message_type import MessageType
 from parsl.process_loggers import wrap_with_logs
 from parsl.serialize import serialize as serialize_object
@@ -53,6 +53,7 @@ def __init__(self,
                  logging_level: int,
                  poll_period: int,
                  cert_dir: Optional[str],
+                 manager_selector: ManagerSelector,
                  ) -> None:
         """
         Parameters
@@ -160,6 +161,8 @@ def __init__(self,
 
         self.heartbeat_threshold = heartbeat_threshold
 
+        self.manager_selector = manager_selector
+
         self.current_platform = {'parsl_v': PARSL_VERSION,
                                  'python_v': "{}.{}.{}".format(sys.version_info.major,
                                                                sys.version_info.minor,
@@ -485,8 +488,7 @@ def process_tasks_to_send(self, interesting_managers: Set[bytes]) -> None:
             interesting=len(interesting_managers)))
 
         if interesting_managers and not self.pending_task_queue.empty():
-            shuffled_managers = list(interesting_managers)
-            random.shuffle(shuffled_managers)
+            shuffled_managers = self.manager_selector.sort_managers(self._ready_managers, interesting_managers)
 
             while shuffled_managers and not self.pending_task_queue.empty():  # cf. the if statement above...
                 manager_id = shuffled_managers.pop()

parsl/executors/high_throughput/manager_selector.py

@@ -0,0 +1,25 @@
+import random
+from abc import ABCMeta, abstractmethod
+from typing import Dict, List, Set
+
+from parsl.executors.high_throughput.manager_record import ManagerRecord
+
+
+class ManagerSelector(metaclass=ABCMeta):
+
+    @abstractmethod
+    def sort_managers(self, ready_managers: Dict[bytes, ManagerRecord], manager_list: Set[bytes]) -> List[bytes]:
+        """ Sort a given list of managers.
+
+        Any operations pertaining to the sorting and rearrangement of the
+        interesting_managers Set should be performed here.
+        """
+        pass
+
+
+class RandomManagerSelector(ManagerSelector):
+
+    def sort_managers(self, ready_managers: Dict[bytes, ManagerRecord], manager_list: Set[bytes]) -> List[bytes]:
+        c_manager_list = list(manager_list)
+        random.shuffle(c_manager_list)
+        return c_manager_list

parsl/tests/test_htex/test_zmq_binding.py

@@ -9,6 +9,7 @@
 
 from parsl import curvezmq
 from parsl.executors.high_throughput.interchange import Interchange
+from parsl.executors.high_throughput.manager_selector import RandomManagerSelector
 
 
 def make_interchange(*, interchange_address: Optional[str], cert_dir: Optional[str]) -> Interchange:
@@ -23,6 +24,7 @@ def make_interchange(*, interchange_address: Optional[str], cert_dir: Optional[s
                        heartbeat_threshold=60,
                        logdir=".",
                        logging_level=logging.INFO,
+                       manager_selector=RandomManagerSelector(),
                        poll_period=10)
 
 

parsl/tests/test_mpi_apps/test_mpiex.py

@@ -44,7 +44,7 @@ def test_init():
 
     new_kwargs = {'max_workers_per_block'}
     excluded_kwargs = {'available_accelerators', 'enable_mpi_mode', 'cores_per_worker', 'max_workers_per_node',
-                       'mem_per_worker', 'cpu_affinity', 'max_workers'}
+                       'mem_per_worker', 'cpu_affinity', 'max_workers', 'manager_selector'}
 
     # Get the kwargs from both HTEx and MPIEx
     htex_kwargs = set(signature(HighThroughputExecutor.__init__).parameters)