ensure trackers are used only on master process (#578)

Signed-off-by: Dushyant Behl <dushyantbehl@in.ibm.com>

Author: dushyantbehl

tuning/trackers/aimstack_tracker.py

@@ -29,6 +29,7 @@ class RunIDExporterAimCallback(AimCallback):
     as soon as it is created, which is during on_init_end.
     """
 
+    _use_aimstack = True
     # path where we export run hash generated by Aim
     # This is used to link back to the expriments from outside aimstack
     run_id_export_path = None
@@ -62,11 +63,15 @@ def on_init_end(self, args, state, control, **kwargs):
         """
         super().on_init_end(args, state, control, **kwargs)
 
+        if not self._use_aimstack or not state.is_world_process_zero:
+            return
+
         if not self._run:
             self.logger.warning(
                 "Aim tracker was requested but did not get initialized;"
                 + " Please check the config"
             )
+            self._use_aimstack = False
             return
 
         if self.tracker:

tuning/trackers/clearml_tracker.py

@@ -33,6 +33,7 @@ class RunURIExporterClearMLCallback(ClearMLCallback):
     as soon as it is created, which is on setup().
     """
 
+    _use_clearml = True
     run_uri_export_path: str = None
     logger = None
     tracker: Tracker = None
@@ -80,11 +81,15 @@ def setup(self, args, state, model, processing_class, **kwargs):
             self._clearml = None
             return
 
+        if not self._use_clearml or not state.is_world_process_zero:
+            return
+
         if not self._initialized or self._clearml_task is None:
             self.logger.warning(
                 "ClearMLtracker was requested but did not get initialized;"
                 + " Please check the config"
             )
+            self._use_clearml = False
             return
 
         task = self._clearml.Task.current_task()

tuning/trackers/mlflow_tracker.py

@@ -30,6 +30,7 @@ class RunURIExporterMlflowCallback(MLflowCallback):
     as soon as it is created, which is on setup().
     """
 
+    _use_mlflow = True
     run_uri_export_path: str = None
     tracker: Tracker
     client = None
@@ -60,11 +61,15 @@ def setup(self, args, state, model):
         """
         super().setup(args, state, model)
 
+        if not self._use_mlflow or not state.is_world_process_zero:
+            return
+
         if not self._initialized:
             self.logger.warning(
                 "mlflow tracker was requested but did not get initialized;"
                 + " Please check the config"
             )
+            self._use_mlflow = False
             return
 
         self.client = self._ml_flow