Fix#317 (#484)

* .

* .

* AutoSklearnClassifier/Regressor's fit, refit, fit_ensemble now return self.

* Initial commit. Work in Progress.

* Fix minor printing error in sprint_statistics.

* Revert "Fix#460"

* first commit. work in progress.

* .

* Autosklearn raises exception if it fails to create tmp, output dirs.

* Safeguarding against Auto-sklearn's file creation and deletion.

* .

* .

* /

* .

* Debugging.

* .

* .

* .

* .

* Fix Tests & Add shared_mode checking

* .

* .

* Minor changes for passing Codacy check.

* Added suggested changes

* .

Author: ahn1340

autosklearn/automl.py

@@ -145,9 +145,7 @@ def fit(
         dataset_name=None,
         only_return_configuration_space=False,
     ):
-        if not self._shared_mode:
-            self._backend.context.delete_directories()
-        else:
+        if self._shared_mode:
             # If this fails, it's likely that this is the first call to get
             # the data manager
             try:
@@ -156,8 +154,6 @@ def fit(
             except IOError:
                 pass
 
-        self._backend.context.create_directories()
-
         if dataset_name is None:
             dataset_name = hash_array_or_matrix(X)
 

autosklearn/estimators.py

@@ -217,7 +217,8 @@ def build_automl(self):
         backend = create(temporary_directory=self.tmp_folder,
                          output_directory=self.output_folder,
                          delete_tmp_folder_after_terminate=self.delete_tmp_folder_after_terminate,
-                         delete_output_folder_after_terminate=self.delete_output_folder_after_terminate)
+                         delete_output_folder_after_terminate=self.delete_output_folder_after_terminate,
+                         shared_mode = self.shared_mode)
         automl = self._get_automl_class()(
             backend=backend,
             time_left_for_this_task=self.time_left_for_this_task,

autosklearn/util/backend.py

@@ -18,10 +18,12 @@
 def create(temporary_directory,
            output_directory,
            delete_tmp_folder_after_terminate=True,
-           delete_output_folder_after_terminate=True):
+           delete_output_folder_after_terminate=True,
+           shared_mode=False):
     context = BackendContext(temporary_directory, output_directory,
                              delete_tmp_folder_after_terminate,
-                             delete_output_folder_after_terminate)
+                             delete_output_folder_after_terminate,
+                             shared_mode)
     backend = Backend(context)
 
     return backend
@@ -33,20 +35,35 @@ def __init__(self,
                  temporary_directory,
                  output_directory,
                  delete_tmp_folder_after_terminate,
-                 delete_output_folder_after_terminate):
-        self._prepare_directories(temporary_directory, output_directory)
+                 delete_output_folder_after_terminate,
+                 shared_mode=False):
+
+        # Check that the names of tmp_dir and output_dir is not the same.
+        if temporary_directory == output_directory \
+            and temporary_directory is not None:
+            raise ValueError("The temporary and the output directory "
+                             "must be different.")
+
         self.delete_tmp_folder_after_terminate = delete_tmp_folder_after_terminate
         self.delete_output_folder_after_terminate = delete_output_folder_after_terminate
+        self.shared_mode = shared_mode
+        # attributes to check that directories were created by autosklearn.
+        self._tmp_dir_created = False
+        self._output_dir_created = False
+
+        self._prepare_directories(temporary_directory, output_directory)
         self._logger = logging.get_logger(__name__)
         self.create_directories()
 
     @property
     def output_directory(self):
-        return self.__output_directory
+        # make sure that tilde does not appear on the path.
+        return os.path.expanduser(os.path.expandvars(self.__output_directory))
 
     @property
     def temporary_directory(self):
-        return self.__temporary_directory
+        # make sure that tilde does not appear on the path.
+        return os.path.expanduser(os.path.expandvars(self.__temporary_directory))
 
     def _prepare_directories(self, temporary_directory, output_directory):
         random_number = random.randint(0, 10000)
@@ -61,20 +78,39 @@ def _prepare_directories(self, temporary_directory, output_directory):
             else '/tmp/autosklearn_output_%d_%d' % (pid, random_number)
 
     def create_directories(self):
-        try:
+        if self.shared_mode:
+            # If shared_mode == True, the tmp and output dir will be shared
+            # by different instances of auto-sklearn.
+            try:
+                os.makedirs(self.temporary_directory)
+            except OSError:
+                pass
+            try:
+                os.makedirs(self.output_directory)
+            except OSError:
+                pass
+
+        else:
+            # Exception is raised if self.temporary_directory already exists.
             os.makedirs(self.temporary_directory)
-        except OSError:
-            pass
-        try:
+            self._tmp_dir_created = True
+
+            # Exception is raised if self.output_directory already exists.
             os.makedirs(self.output_directory)
-        except OSError:
-            pass
+            self._output_dir_created = True
+
 
     def __del__(self):
         self.delete_directories(force=False)
 
     def delete_directories(self, force=True):
         if self.delete_output_folder_after_terminate or force:
+            if self._output_dir_created is False and self.shared_mode is False:
+                raise ValueError("Failed to delete output dir: %s "
+                              "because auto-sklearn did not create it. "
+                              "Please make sure that the specified output "
+                              "dir does not exist when instantiating "
+                              "auto-sklearn." % self.output_directory)
             try:
                 shutil.rmtree(self.output_directory)
             except Exception:
@@ -86,6 +122,12 @@ def delete_directories(self, force=True):
                           self.output_directory)
 
         if self.delete_tmp_folder_after_terminate or force:
+            if self._tmp_dir_created is False and self.shared_mode is False:
+                raise ValueError("Failed to delete tmp dir: % s "
+                              "because auto-sklearn did not create it. "
+                              "Please make sure that the specified tmp "
+                              "dir does not exist when instantiating "
+                              "auto-sklearn." % self.temporary_directory)
             try:
                 shutil.rmtree(self.temporary_directory)
             except Exception:
@@ -418,4 +460,4 @@ def write_txt_file(self, filepath, data, name):
                 self.logger.debug('Created %s file %s' % (name, filepath))
             else:
                 self.logger.debug('%s file already present %s' %
-                                  (name, filepath))
+                                  (name, filepath))

test/test_automl/base.py

@@ -3,6 +3,7 @@
 import shutil
 import time
 import unittest
+from autosklearn.util.backend import create
 
 
 class Base(unittest.TestCase):
@@ -19,25 +20,33 @@ def setUp(self):
         except Exception:
             self.travis = False
 
-    def _setUp(self, output):
-        if os.path.exists(output):
+    def _setUp(self, dir):
+        if os.path.exists(dir):
             for i in range(10):
                 try:
-                    shutil.rmtree(output)
+                    shutil.rmtree(dir)
                     break
                 except OSError:
                     time.sleep(1)
-        try:
-            os.makedirs(output)
-        except OSError:
-            pass
 
+    def _create_backend(self, test_name):
+        tmp = os.path.join(self.test_dir, '..', '.tmp._%s' % test_name)
+        output = os.path.join(self.test_dir, '..', '.output._%s' % test_name)
+        # Make sure the folders we wanna create do not already exist.
+        self._setUp(tmp)
+        self._setUp(output)
+        backend = create(tmp, output)
+        return backend
 
-    def _tearDown(self, output):
-        if os.path.exists(output):
+    def _tearDown(self, dir):
+        """
+        Delete the temporary and the output directories manually
+        in case they are not deleted.
+        """
+        if os.path.exists(dir):
             for i in range(10):
                 try:
-                    shutil.rmtree(output)
+                    shutil.rmtree(dir)
                     break
                 except OSError:
                     time.sleep(1)

test/test_automl/test_automl.py

@@ -43,14 +43,12 @@ def setUp(self):
         self.automl._delete_output_directories = lambda: 0
 
     def test_refit_shuffle_on_fail(self):
-        output = os.path.join(self.test_dir, '..', '.tmp_refit_shuffle_on_fail')
-        context = BackendContext(output, output, False, False)
-        backend = Backend(context)
+        backend_api = self._create_backend('test_refit_shuffle_on_fail')
 
         failing_model = unittest.mock.Mock()
         failing_model.fit.side_effect = [ValueError(), ValueError(), None]
 
-        auto = AutoML(backend, 20, 5)
+        auto = AutoML(backend_api, 20, 5)
         ensemble_mock = unittest.mock.Mock()
         auto.ensemble_ = ensemble_mock
         ensemble_mock.get_selected_model_identifiers.return_value = [1]
@@ -63,6 +61,10 @@ def test_refit_shuffle_on_fail(self):
 
         self.assertEqual(failing_model.fit.call_count, 3)
 
+        del auto
+        self._tearDown(backend_api.temporary_directory)
+        self._tearDown(backend_api.output_directory)
+
     def test_only_loads_ensemble_models(self):
         identifiers = [(1, 2), (3, 4)]
 
@@ -96,11 +98,9 @@ def test_raises_if_no_models(self):
         self.automl._load_models()
 
     def test_fit(self):
-        output = os.path.join(self.test_dir, '..', '.tmp_test_fit')
-        self._setUp(output)
+        backend_api = self._create_backend('test_fit')
 
         X_train, Y_train, X_test, Y_test = putil.get_dataset('iris')
-        backend_api = backend.create(output, output)
         automl = autosklearn.automl.AutoML(backend_api, 20, 5)
         automl.fit(
             X_train, Y_train, metric=accuracy, task=MULTICLASS_CLASSIFICATION,
@@ -110,7 +110,8 @@ def test_fit(self):
         self.assertEqual(automl._task, MULTICLASS_CLASSIFICATION)
 
         del automl
-        self._tearDown(output)
+        self._tearDown(backend_api.temporary_directory)
+        self._tearDown(backend_api.output_directory)
 
     def test_fit_roar(self):
         def get_roar_object_callback(
@@ -129,11 +130,9 @@ def get_roar_object_callback(
                 tae_runner=ta,
             )
 
-        output = os.path.join(self.test_dir, '..', '.tmp_test_fit_roar')
-        self._setUp(output)
+        backend_api = self._create_backend('test_fit_roar')
 
         X_train, Y_train, X_test, Y_test = putil.get_dataset('iris')
-        backend_api = backend.create(output, output)
         automl = autosklearn.automl.AutoML(
             backend=backend_api,
             time_left_for_this_task=20,
@@ -149,16 +148,15 @@ def get_roar_object_callback(
         self.assertEqual(automl._task, MULTICLASS_CLASSIFICATION)
 
         del automl
-        self._tearDown(output)
+        self._tearDown(backend_api.temporary_directory)
+        self._tearDown(backend_api.output_directory)
 
     def test_binary_score_and_include(self):
         """
         Test fix for binary classification prediction
         taking the index 1 of second dimension in prediction matrix
         """
-
-        output = os.path.join(self.test_dir, '..', '.tmp_test_binary_score')
-        self._setUp(output)
+        backend_api = self._create_backend('test_binary_score_and_include')
 
         data = sklearn.datasets.make_classification(
             n_samples=400, n_features=10, n_redundant=1, n_informative=3,
@@ -168,7 +166,6 @@ def test_binary_score_and_include(self):
         X_test = data[0][200:]
         Y_test = data[1][200:]
 
-        backend_api = backend.create(output, output)
         automl = autosklearn.automl.AutoML(backend_api, 20, 5,
                                            include_estimators=['sgd'],
                                            include_preprocessors=['no_preprocessing'])
@@ -182,18 +179,17 @@ def test_binary_score_and_include(self):
         self.assertGreaterEqual(score, 0.4)
 
         del automl
-        self._tearDown(output)
+        self._tearDown(backend_api.temporary_directory)
+        self._tearDown(backend_api.output_directory)
 
     def test_automl_outputs(self):
-        output = os.path.join(self.test_dir, '..',
-                              '.tmp_test_automl_outputs')
-        self._setUp(output)
+        backend_api = self._create_backend('test_automl_outputs')
+
         name = '31_bac'
         dataset = os.path.join(self.test_dir, '..', '.data', name)
-        data_manager_file = os.path.join(output, '.auto-sklearn',
+        data_manager_file = os.path.join(backend_api.temporary_directory, '.auto-sklearn',
                                          'datamanager.pkl')
 
-        backend_api = backend.create(output, output)
         auto = autosklearn.automl.AutoML(
             backend_api, 20, 5,
             initial_configurations_via_metalearning=0,
@@ -212,43 +208,41 @@ def test_automl_outputs(self):
                    'start_time_100', 'datamanager.pkl',
                    'predictions_ensemble',
                    'ensembles', 'predictions_test', 'models']
-        self.assertEqual(sorted(os.listdir(os.path.join(output,
+        self.assertEqual(sorted(os.listdir(os.path.join(backend_api.temporary_directory,
                                                         '.auto-sklearn'))),
                          sorted(fixture))
 
         # At least one ensemble, one validation, one test prediction and one
         # model and one ensemble
-        fixture = os.listdir(os.path.join(output, '.auto-sklearn',
+        fixture = os.listdir(os.path.join(backend_api.temporary_directory, '.auto-sklearn',
                                           'predictions_ensemble'))
         self.assertIn('predictions_ensemble_100_1.npy', fixture)
 
-        fixture = os.listdir(os.path.join(output, '.auto-sklearn',
+        fixture = os.listdir(os.path.join(backend_api.temporary_directory, '.auto-sklearn',
                                           'models'))
         self.assertIn('100.1.model', fixture)
 
-        fixture = os.listdir(os.path.join(output, '.auto-sklearn',
+        fixture = os.listdir(os.path.join(backend_api.temporary_directory, '.auto-sklearn',
                                           'ensembles'))
         self.assertIn('100.0.ensemble', fixture)
 
         # Start time
-        start_time_file_path = os.path.join(output, '.auto-sklearn',
+        start_time_file_path = os.path.join(backend_api.temporary_directory, '.auto-sklearn',
                                             "start_time_100")
         with open(start_time_file_path, 'r') as fh:
             start_time = float(fh.read())
         self.assertGreaterEqual(time.time() - start_time, 10)
 
         del auto
-        self._tearDown(output)
+        self._tearDown(backend_api.temporary_directory)
+        self._tearDown(backend_api.output_directory)
 
     def test_do_dummy_prediction(self):
         for name in ['401_bac', '31_bac', 'adult', 'cadata']:
-            output = os.path.join(self.test_dir, '..',
-                                  '.tmp_test_do_dummy_prediction')
-            self._setUp(output)
+            backend_api = self._create_backend('test_do_dummy_prediction')
 
             dataset = os.path.join(self.test_dir, '..', '.data', name)
 
-            backend_api = backend.create(output, output)
             auto = autosklearn.automl.AutoML(
                 backend_api, 20, 5,
                 initial_configurations_via_metalearning=25)
@@ -260,14 +254,17 @@ def test_do_dummy_prediction(self):
             auto._do_dummy_prediction(D, 1)
 
             # Ensure that the dummy predictions are not in the current working
-            # directory, but in the output directory (under output)
+            # directory, but in the temporary directory.
             self.assertFalse(os.path.exists(os.path.join(os.getcwd(),
                                                          '.auto-sklearn')))
             self.assertTrue(os.path.exists(os.path.join(
-                output, '.auto-sklearn', 'predictions_ensemble',
+                backend_api.temporary_directory, '.auto-sklearn', 'predictions_ensemble',
                 'predictions_ensemble_1_1.npy')))
 
             del auto
-            self._tearDown(output)
+            self._tearDown(backend_api.temporary_directory)
+            self._tearDown(backend_api.output_directory)
 
 
+if __name__=="__main__":
+    unittest.main()