additional output

Author: mfeurer

autosklearn/automl.py

@@ -736,6 +736,7 @@ def fit(
                 self._logger.exception(e)
                 raise
 
+        self._logger.info("Starting shutdown...")
         # Wait until the ensemble process is finished to avoid shutting down
         # while the ensemble builder tries to access the data
         if proc_ensemble is not None:
@@ -750,13 +751,21 @@ def fit(
                 future = proc_ensemble.futures.pop()
                 # Now we need to wait for the future to return as it cannot be cancelled while it
                 # is running: https://stackoverflow.com/a/49203129
+                self._logger.info("Ensemble script still running, waiting for it to finish.")
                 future.result()
+                self._logger.info("Ensemble script finished, continue shutdown.")
 
         if load_models:
+            self._logger.info("Loading models...")
             self._load_models()
+            self._logger.info("Finished loading models...")
+
+        self._logger.info("Closing the dask infrastructure")
         self._close_dask_client()
+        self._logger.info("Finished closing the dask infrastructure")
 
         # Clean up the logger
+        self._logger.info("Starting to clean up the logger")
         self._clean_logger()
 
         return self

autosklearn/ensemble_builder.py

@@ -23,6 +23,7 @@
 from smac.callbacks import IncorporateRunResultCallback
 from smac.optimizer.smbo import SMBO
 from smac.runhistory.runhistory import RunInfo, RunValue
+from smac.tae.base import StatusType
 
 from autosklearn.util.backend import Backend
 from autosklearn.constants import BINARY_CLASSIFICATION
@@ -147,6 +148,8 @@ def __call__(
         result: RunValue,
         time_left: float,
     ):
+        if result.status in (StatusType.STOP, StatusType.ABORT):
+            return
         self.build_ensemble(smbo.tae_runner.client)
 
     def build_ensemble(
@@ -582,6 +585,7 @@ def run(
             if time_left - time_buffer < 1:
                 break
             context = multiprocessing.get_context(pynisher_context)
+
             safe_ensemble_script = pynisher.enforce_limits(
                 wall_time_in_s=int(time_left - time_buffer),
                 mem_in_mb=self.memory_limit,

autosklearn/evaluation/__init__.py

@@ -176,6 +176,7 @@ def __init__(self, backend, autosklearn_seed, resampling_strategy, metric,
             self._get_test_loss = False
 
         self.pynisher_context = pynisher_context
+        self.logger = autosklearn.util.logging_.get_logger("TAE")
 
     def run_wrapper(
         self,
@@ -218,7 +219,12 @@ def run_wrapper(
         if remaining_time - 5 < run_info.cutoff:
             run_info = run_info._replace(cutoff=int(remaining_time - 5))
 
+        config_id = (
+            run_info.config if isinstance(run_info.config, int) else run_info.config.config_id
+        )
+
         if run_info.cutoff < 1.0:
+            self.logger.info("Not starting configuration %d because time is up" % config_id)
             return run_info, RunValue(
                 status=StatusType.STOP,
                 cost=self.worst_possible_result,
@@ -233,6 +239,7 @@ def run_wrapper(
         ):
             run_info = run_info._replace(cutoff=int(np.ceil(run_info.cutoff)))
 
+        self.logger.info("Starting to evaluate configuration %d" % config_id)
         return super().run_wrapper(run_info=run_info)
 
     def run(
@@ -432,11 +439,14 @@ def run(
 
         if isinstance(config, int):
             origin = 'DUMMY'
+            config_id = config
         else:
             origin = getattr(config, 'origin', 'UNKNOWN')
+            config_id = config.config_id
         additional_run_info['configuration_origin'] = origin
 
         runtime = float(obj.wall_clock_time)
 
         autosklearn.evaluation.util.empty_queue(queue)
+        self.logger.info("Finished evaluating configuration %d" % config_id)
         return status, cost, runtime, additional_run_info