Improve data handling (#994)

* Improve managing disk space

* fixes a bug in the ensemble builder that would cause the ensemble building to break when giving a limit on the disk space to use
* allow more fine-grained control over what files to save on disk
* make the output directory optional and only create it if it is actively passed in by the user

* fix bug in logging function

* Improve cv models directories (#993)

* restructure directories for models and predictions

* pep8 and mypy

* fix tests, include offline feedback

* update unittests after rebase

* add forgotten files

* add forgotten file

* fix tests

* fix merge issues

* fix unit tests

* minor improvements

* remove print statement

Author: mfeurer

autosklearn/automl.py

@@ -37,6 +37,7 @@
 from autosklearn.evaluation.abstract_evaluator import _fit_and_suppress_warnings
 from autosklearn.evaluation.train_evaluator import _fit_with_budget
 from autosklearn.metrics import calculate_score
+from autosklearn.util.backend import Backend
 from autosklearn.util.stopwatch import StopWatch
 from autosklearn.util.logging_ import get_logger, setup_logger
 from autosklearn.util import pipeline, RE_PATTERN
@@ -95,7 +96,7 @@ def send_warnings_to_log(
 class AutoML(BaseEstimator):
 
     def __init__(self,
-                 backend,
+                 backend: Backend,
                  time_left_for_this_task,
                  per_run_time_limit,
                  initial_configurations_via_metalearning=25,
@@ -179,7 +180,7 @@ def __init__(self,
             raise ValueError('disable_evaluator_output must be of type bool '
                              'or list.')
         if isinstance(self._disable_evaluator_output, list):
-            allowed_elements = ['model', 'y_optimization']
+            allowed_elements = ['model', 'cv_model', 'y_optimization', 'y_test', 'y_valid']
             for element in self._disable_evaluator_output:
                 if element not in allowed_elements:
                     raise ValueError("List member '%s' for argument "
@@ -313,7 +314,7 @@ def _do_dummy_prediction(self, datamanager, num_run):
                                     cost_for_crash=get_cost_of_crash(self._metric),
                                     **self._resampling_strategy_arguments)
 
-        status, cost, runtime, additional_info = ta.run(1, cutoff=self._time_for_task)
+        status, cost, runtime, additional_info = ta.run(num_run, cutoff=self._time_for_task)
         if status == StatusType.SUCCESS:
             self._logger.info("Finished creating dummy predictions.")
         else:
@@ -511,14 +512,6 @@ def fit(
         )
 
         self._backend._make_internals_directory()
-        try:
-            os.makedirs(self._backend.get_model_dir())
-        except (OSError, FileExistsError):
-            raise
-        try:
-            os.makedirs(self._backend.get_cv_model_dir())
-        except (OSError, FileExistsError):
-            raise
 
         self._task = datamanager.info['task']
         self._label_num = datamanager.info['label_num']
@@ -942,9 +935,9 @@ def _load_best_individual_model(self):
         # SingleBest contains the best model found by AutoML
         ensemble = SingleBest(
             metric=self._metric,
-            random_state=self._seed,
+            seed=self._seed,
             run_history=self.runhistory_,
-            model_dir=self._backend.get_model_dir(),
+            backend=self._backend,
         )
         self._logger.warning(
             "No valid ensemble was created. Please check the log"