Merge pull request #167 from Ayaro/development

Fix parallel example exception

Author: mfeurer

autosklearn/automl.py

@@ -471,7 +471,7 @@ def predict(self, X):
         predictions = self.ensemble_.predict(all_predictions)
         return predictions
 
-    def fit_ensemble(self, task=None, metric=None, precision='32',
+    def fit_ensemble(self, y, task=None, metric=None, precision='32',
                      dataset_name=None, ensemble_nbest=None,
                      ensemble_size=None):
         if self._logger is None:
@@ -490,16 +490,33 @@ def _get_ensemble_process(self, time_left_for_ensembles,
 
         if task is None:
             task = self._task
+        else:
+            self._task = task
+
         if metric is None:
             metric = self._metric
+        else:
+            self._metric = metric
+
         if precision is None:
             precision = self.precision
+        else:
+            self.precision = precision
+
         if dataset_name is None:
             dataset_name = self._dataset_name
+        else:
+            self._dataset_name = dataset_name
+
         if ensemble_nbest is None:
             ensemble_nbest = self._ensemble_nbest
+        else:
+            self._ensemble_nbest = ensemble_nbest
+
         if ensemble_size is None:
             ensemble_size = self._ensemble_size
+        else:
+            self._ensemble_size = ensemble_size
 
         return EnsembleBuilder(backend=self._backend,
                                dataset_name=dataset_name,

autosklearn/estimators.py

@@ -47,7 +47,7 @@ def refit(self, X, y):
         """
         return self._automl.refit(X, y)
 
-    def fit_ensemble(self, task=None, metric=None, precision='32',
+    def fit_ensemble(self, y, task=None, metric=None, precision='32',
                      dataset_name=None, ensemble_nbest=None,
                      ensemble_size=None):
         """Build the ensemble.
@@ -58,7 +58,7 @@ def fit_ensemble(self, task=None, metric=None, precision='32',
         -------
         self
         """
-        return self._automl.fit_ensemble(task, metric, precision,
+        return self._automl.fit_ensemble(y, task, metric, precision,
                                          dataset_name, ensemble_nbest,
                                          ensemble_size)
 
@@ -253,12 +253,12 @@ def fit(self, *args, **kwargs):
         self._automl = self.build_automl()
         super(AutoSklearnEstimator, self).fit(*args, **kwargs)
 
-    def fit_ensemble(self, task=None, metric=None, precision='32',
+    def fit_ensemble(self, y, task=None, metric=None, precision='32',
                      dataset_name=None, ensemble_nbest=None,
                      ensemble_size=None):
         if self._automl is None:
             self._automl = self.build_automl()
-        return self._automl.fit_ensemble(task, metric, precision,
+        return self._automl.fit_ensemble(y, task, metric, precision,
                                          dataset_name, ensemble_nbest,
                                          ensemble_size)
 
@@ -426,6 +426,27 @@ def fit(self, X, y,
                                       force_all_finite=False)
         if scipy.sparse.issparse(X):
             X.sort_indices()
+
+        y = self._process_target_classes(y)
+
+        if self._n_outputs > 1:
+            task = MULTILABEL_CLASSIFICATION
+        else:
+            if len(self._classes[0]) == 2:
+                task = BINARY_CLASSIFICATION
+            else:
+                task = MULTICLASS_CLASSIFICATION
+
+        return self._automl.fit(X, y, task, metric, feat_type, dataset_name)
+
+    def fit_ensemble(self, y, task=None, metric=None, precision='32',
+                     dataset_name=None, ensemble_nbest=None,
+                     ensemble_size=None):
+        self._process_target_classes(y)
+        return self._automl.fit_ensemble(y, task, metric, precision, dataset_name,
+                                         ensemble_nbest, ensemble_size)
+
+    def _process_target_classes(self, y):
         y = np.atleast_1d(y)
         if y.ndim == 2 and y.shape[1] == 1:
             warnings.warn("A column-vector y was passed when a 1d array was"
@@ -452,19 +473,12 @@ def fit(self, X, y,
 
         self._n_classes = np.array(self._n_classes, dtype=np.int)
 
-        if self._n_outputs > 1:
-            task = MULTILABEL_CLASSIFICATION
-        else:
-            if len(self._classes[0]) == 2:
-                task = BINARY_CLASSIFICATION
-            else:
-                task = MULTICLASS_CLASSIFICATION
-
         # TODO: fix metafeatures calculation to allow this!
         if y.shape[1] == 1:
             y = y.flatten()
 
-        return self._automl.fit(X, y, task, metric, feat_type, dataset_name)
+        return y
+
 
     def predict(self, X):
         predicted_probabilities = self._automl.predict(X)

example/example_parallel.py

@@ -1,17 +1,23 @@
 # -*- encoding: utf-8 -*-
 import multiprocessing
-
 import numpy as np
-
+import shutil
 import sklearn.datasets
 import sklearn.metrics
-
 from autosklearn.classification import AutoSklearnClassifier
 from autosklearn.constants import *
 
 tmp_folder = '/tmp/autosklearn_parallel_example_tmp'
 output_folder = '/tmp/autosklearn_parallel_example_out'
 
+try:
+    shutil.rmtree(tmp_folder)
+except OSError as e:
+    pass
+try:
+    shutil.rmtree(output_folder)
+except OSError:
+    pass
 
 def spawn_classifier(seed, dataset_name):
     """Spawn a subprocess.
@@ -84,7 +90,8 @@ def spawn_classifier(seed, dataset_name):
 
     # Both the ensemble_size and ensemble_nbest parameters can be changed now if
     # necessary
-    automl.fit_ensemble(task=MULTICLASS_CLASSIFICATION,
+    automl.fit_ensemble(y_train,
+                        task=MULTICLASS_CLASSIFICATION,
                         metric=ACC_METRIC,
                         precision='32',
                         dataset_name='digits',

test/test_automl/test_estimators.py

@@ -5,6 +5,8 @@
 import sys
 import unittest
 
+import sklearn
+
 try:
     import mock
 except ImportError:
@@ -17,7 +19,7 @@
 import autosklearn.pipeline.util as putil
 from autosklearn.classification import AutoSklearnClassifier
 from autosklearn.estimators import AutoMLClassifier
-from autosklearn.util.backend import Backend
+from autosklearn.util.backend import Backend, BackendContext
 from autosklearn.constants import *
 
 sys.path.append(os.path.dirname(__file__))
@@ -93,13 +95,16 @@ def test_feat_type_wrong_arguments(self):
                                 cls.fit,
                                 X=X, y=y, feat_type=['Car']*100)
 
-    @unittest.skip("pSMAC not yet working with new python SMAC")
     def test_fit_pSMAC(self):
         output = os.path.join(self.test_dir, '..', '.tmp_estimator_fit_pSMAC')
         self._setUp(output)
 
         X_train, Y_train, X_test, Y_test = putil.get_dataset('iris')
 
+        # test parallel Classifier to predict classes, not only indexes
+        Y_train = Y_train + 1
+        Y_test = Y_test + 1
+
         automl = AutoSklearnClassifier(time_left_for_this_task=15,
                                        per_run_time_limit=15,
                                        output_folder=output,
@@ -128,10 +133,11 @@ def test_fit_pSMAC(self):
 
         probas_test = np.zeros((len(Y_test), 3), dtype=float)
         for i, value in enumerate(Y_test):
-            probas_test[i, value] = 1.0
+            probas_test[i, value - 1] = 1.0
 
         dummy = ArrayReturningDummyPredictor(probas_test)
-        backend = Backend(output, output)
+        context = BackendContext(output, output, False, False)
+        backend = Backend(context)
         backend.save_model(dummy, 30, 1)
 
         automl = AutoSklearnClassifier(time_left_for_this_task=15,
@@ -142,16 +148,26 @@ def test_fit_pSMAC(self):
                                        seed=2,
                                        initial_configurations_via_metalearning=0,
                                        ensemble_size=0)
-        automl.fit(X_train, Y_train)
-        automl.run_ensemble_builder(0, 1, 50).wait()
+        automl.fit_ensemble(Y_train,
+                            task=MULTICLASS_CLASSIFICATION,
+                            metric=ACC_METRIC,
+                            precision='32',
+                            dataset_name='iris',
+                            ensemble_size=20,
+                            ensemble_nbest=50)
 
-        score = automl.score(X_test, Y_test)
+        predictions = automl.predict(X_test)
+        score = sklearn.metrics.accuracy_score(Y_test, predictions)
 
         self.assertEqual(len(os.listdir(os.path.join(output, '.auto-sklearn',
                                                      'ensembles'))), 1)
         self.assertGreaterEqual(score, 0.90)
         self.assertEqual(automl._automl._automl._task, MULTICLASS_CLASSIFICATION)
 
+        models = automl._automl._automl.models_
+        classifier_types = [type(c) for c in models.values()]
+        self.assertIn(ArrayReturningDummyPredictor, classifier_types)
+
         del automl
         self._tearDown(output)