Issue#1100 Provide argument in AutoSklearnClassifier and AutoSklearnRegressor to pass callback function (#1140)

* Fix ASKL2 link in the docs

* implementation for including callback function after each trial

* Added Typing information to get_trials_callback in AutoSklearnEstimator init

* added get_trials_callback in docstring, used tempfile
in test cases

* removed delete_output_folder_after_terminate while creating AutoSklearnClassifier instance from the test case

* added get_trials_callback in test_check_askl2_same_arguments_as_askl, changed code formatting

* removed unwanted imports

Co-authored-by: Matthias Feurer <[email protected]>
Co-authored-by: Shubhmay Potdar <[email protected]>

Author: 3 people

autosklearn/automl.py

@@ -139,7 +139,8 @@ def __init__(self,
                  smac_scenario_args=None,
                  logging_config=None,
                  metric=None,
-                 scoring_functions=None
+                 scoring_functions=None,
+                 get_trials_callback=None
                  ):
         super(AutoML, self).__init__()
         self.configuration_space = None
@@ -181,6 +182,7 @@ def __init__(self,
                                      "'disable_evaluator_output' must be one "
                                      "of " + str(allowed_elements))
         self._get_smac_object_callback = get_smac_object_callback
+        self._get_trials_callback = get_trials_callback
         self._smac_scenario_args = smac_scenario_args
         self.logging_config = logging_config
 
@@ -758,6 +760,7 @@ def fit(
                 port=self._logger_port,
                 pynisher_context=self._multiprocessing_context,
                 ensemble_callback=proc_ensemble,
+                trials_callback=self._get_trials_callback
             )
 
             try:

autosklearn/estimators.py

@@ -50,6 +50,7 @@ def __init__(
         metric=None,
         scoring_functions: Optional[List[Scorer]] = None,
         load_models: bool = True,
+        get_trials_callback=None
     ):
         """
         Parameters
@@ -223,6 +224,12 @@ def __init__(
 
         load_models : bool, optional (True)
             Whether to load the models after fitting Auto-sklearn.
+           
+        get_trials_callback: callable
+            Callback function to create an object of subclass defined in module
+            `smac.callbacks <https://automl.github.io/SMAC3/master/apidoc/smac.callbacks.html>`_.
+            This is an advanced feature. Use only if you are familiar with
+            `SMAC <https://automl.github.io/SMAC3/master/index.html>`_.
 
         Attributes
         ----------
@@ -264,6 +271,7 @@ def __init__(
         self.metric = metric
         self.scoring_functions = scoring_functions
         self.load_models = load_models
+        self.get_trials_callback = get_trials_callback
 
         self.automl_ = None  # type: Optional[AutoML]
 
@@ -314,7 +322,8 @@ def build_automl(self):
             logging_config=self.logging_config,
             metadata_directory=self.metadata_directory,
             metric=self.metric,
-            scoring_functions=self.scoring_functions
+            scoring_functions=self.scoring_functions,
+            get_trials_callback=self.get_trials_callback
         )
 
         return automl

autosklearn/smbo.py

@@ -18,6 +18,7 @@
 from smac.scenario.scenario import Scenario
 from smac.tae.serial_runner import SerialRunner
 from smac.tae.dask_runner import DaskParallelRunner
+from smac.callbacks import IncorporateRunResultCallback
 
 
 import autosklearn.metalearning
@@ -241,6 +242,7 @@ def __init__(self, config_space, dataset_name,
                  scoring_functions=None,
                  pynisher_context='spawn',
                  ensemble_callback: typing.Optional[EnsembleBuilderManager] = None,
+                 trials_callback: typing.Optional[IncorporateRunResultCallback] = None
                  ):
         super(AutoMLSMBO, self).__init__()
         # data related
@@ -288,6 +290,7 @@ def __init__(self, config_space, dataset_name,
         self.pynisher_context = pynisher_context
 
         self.ensemble_callback = ensemble_callback
+        self.trials_callback = trials_callback
 
         dataset_name_ = "" if dataset_name is None else dataset_name
         logger_name = '%s(%d):%s' % (self.__class__.__name__, self.seed, ":" + dataset_name_)
@@ -499,6 +502,8 @@ def run_smbo(self):
 
         if self.ensemble_callback is not None:
             smac.register_callback(self.ensemble_callback)
+        if self.trials_callback is not None:
+            smac.register_callback(self.trials_callback)
 
         smac.optimize()
 

doc/index.rst

@@ -97,7 +97,7 @@ reference to the following paper:
 If you are using Auto-sklearn 2.0, please also cite
 
 
- Auto-Sklearn 2.0: The Next Generation, Feurer *et al.*, to appear (2020).
+ `Auto-Sklearn 2.0: The Next Generation <https://arxiv.org/abs/2007.04074>`_, Feurer *et al.*, (arXiv, 2020).
 
  Bibtex entry::
 
@@ -107,7 +107,7 @@ If you are using Auto-sklearn 2.0, please also cite
                   Falkner, Stefan and Lindauer, Marius and Hutter, Frank},
         booktitle = {Advances in Neural Information Processing Systems 28},
         year = {2020},
-        journal = {arXiv:2006.???? [cs.LG]},
+        journal = {arXiv:2007.04074 [cs.LG]},
      }
 
 Contributing

test/test_automl/test_estimators.py

@@ -700,7 +700,8 @@ def test_check_askl2_same_arguments_as_askl():
                            'get_smac_object_callback',
                            'initial_configurations_via_metalearning',
                            'resampling_strategy',
-                           'metadata_directory']
+                           'metadata_directory',
+                           'get_trials_callback']
     unexpected_args = set(extra_arguments) - set(expected_extra_args)
     assert len(unexpected_args) == 0, unexpected_args
 

test/test_util/test_trials_callback.py

@@ -0,0 +1,68 @@
+import os
+import tempfile
+import unittest
+
+import pandas as pd
+from smac.callbacks import IncorporateRunResultCallback
+from smac.optimizer.smbo import SMBO
+from smac.runhistory.runhistory import RunInfo, RunValue
+from smac.tae.base import StatusType
+
+import autosklearn.pipeline.util as putil
+from autosklearn.classification import AutoSklearnClassifier
+
+
+class AutoMLTrialsCallBack(IncorporateRunResultCallback):
+
+    def __init__(self, fname):
+        self.trials_num = 1
+        self.fname = fname
+        with open(fname, "w") as fp:
+            fp.write("TrialNo, "
+                     "StartTime, "
+                     "EndTime, "
+                     "Status, "
+                     "TrainLoss, "
+                     "ValidLoss, "
+                     "TestLoss, "
+                     "Classifier")
+
+    def __call__(
+            self, smbo: 'SMBO',
+            run_info: RunInfo,
+            result: RunValue,
+            time_left: float,
+    ) -> None:
+        train_loss, valid_loss, test_loss = None, None, None
+        trial_start_time = result.starttime
+        trial_end_time = result.endtime
+        trial_status = result.status.name
+        if trial_status == StatusType.SUCCESS.name:
+            train_loss = result.additional_info.get('train_loss')
+            valid_loss = result.cost
+            test_loss = result.additional_info.get('test_loss')
+        trial_classifier = run_info.config.get_dictionary()['classifier:__choice__']
+        with open(self.fname, "a+") as fp:
+            fp.write(f"\n {self.trials_num}, {trial_start_time}, {trial_end_time}, {trial_status}, "
+                     f"{train_loss}, {valid_loss}, {test_loss}, {trial_classifier}")
+        self.trials_num += 1
+
+
+class VerifyTrialsCallBack(unittest.TestCase):
+
+    def test_trials_callback_execution(self):
+        trials_summary_fname = os.path.join(tempfile.gettempdir(), "trials.csv")
+        X_train, Y_train, X_test, Y_test = putil.get_dataset('breast_cancer')
+        cls = AutoSklearnClassifier(time_left_for_this_task=30,
+                                    initial_configurations_via_metalearning=0,
+                                    per_run_time_limit=10,
+                                    memory_limit=1024,
+                                    delete_tmp_folder_after_terminate=False,
+                                    n_jobs=1,
+                                    include_estimators=["sgd"],
+                                    include_preprocessors=["no_preprocessing"],
+                                    get_trials_callback=AutoMLTrialsCallBack(trials_summary_fname)
+                                    )
+        cls.fit(X_train, Y_train, X_test, Y_test)
+        trials = pd.read_csv(trials_summary_fname)
+        assert trials.shape[0] > 0, f"Auto-Sklearn explored {trials.shape[0] - 1} trials"