Merge pull request #37 from automl/development

Development

Author: mfeurer

autosklearn/automl.py

@@ -187,7 +187,8 @@ def __init__(self,
         self._metric = None
         self._label_num = None
         self.models_ = None
-        self.ensemble_indices_ = None
+        self.ensemble_ = None
+        self._can_predict = False
 
         self._debug_mode = debug_mode
         self._backend = Backend(self._output_dir, self._tmp_dir)
@@ -242,9 +243,14 @@ def fit(self, X, y,
             raise ValueError('Array feat_type does not have same number of '
                              'variables as X has features. %d vs %d.' %
                              (len(feat_type), X.shape[1]))
-        if feat_type is not None and not all([isinstance(f, bool)
+        if feat_type is not None and not all([isinstance(f, str)
                                               for f in feat_type]):
-            raise ValueError('Array feat_type must only contain bools.')
+            raise ValueError('Array feat_type must only contain strings.')
+        if feat_type is not None:
+            for ft in feat_type:
+                if ft.lower() not in ['categorical', 'numerical']:
+                    raise ValueError('Only `Categorical` and `Numerical` are '
+                                     'valid feature types, you passed `%s`' % ft)
 
         loaded_data_manager = XYDataManager(X, y,
                                             task=task,
@@ -298,16 +304,19 @@ def _print_load_time(basename, time_left_for_this_task,
         return time_for_load_data
 
     def _do_dummy_prediction(self, datamanager):
+        self._logger.info("Starting to create dummy predictions.")
         autosklearn.cli.base_interface.main(datamanager,
                                             self._resampling_strategy,
                                             None,
                                             None,
-                                            mode_args=self._resampling_strategy_arguments)
+                                            mode_args=self._resampling_strategy_arguments,
+                                            output_dir=self._tmp_dir)
+        self._logger.info("Finished creating dummy predictions.")
 
     def _fit(self, datamanager):
         # Reset learnt stuff
         self.models_ = None
-        self.ensemble_indices_ = None
+        self.ensemble_ = None
 
         # Check arguments prior to doing anything!
         if self._resampling_strategy not in ['holdout', 'holdout-iterative-fit',
@@ -352,7 +361,8 @@ def _fit(self, datamanager):
                 self._logger)
 
         # == Perform dummy predictions
-        self._do_dummy_prediction(datamanager)
+        if self._resampling_strategy in ['holdout', 'holdout-iterative-fit']:
+            self._do_dummy_prediction(datamanager)
 
         # = Create a searchspace
         # Do this before One Hot Encoding to make sure that it creates a
@@ -371,6 +381,12 @@ def _fit(self, datamanager):
             self._include_preprocessors)
         self.configuration_space_created_hook(datamanager)
 
+        # == RUN ensemble builder
+        # Do this before calculating the meta-features to make sure that the
+        # dummy predictions are actually included in the ensemble even if
+        # calculating the meta-features takes very long
+        proc_ensembles = self.run_ensemble_builder()
+
         # == Calculate metafeatures
         meta_features = _calculate_metafeatures(
             data_feat_type=datamanager.feat_type,
@@ -481,9 +497,6 @@ def _fit(self, datamanager):
                              resampling_strategy_arguments=self._resampling_strategy_arguments,
                              shared_mode=self._shared_mode)
 
-        # == RUN ensemble builder
-        proc_ensembles = self.run_ensemble_builder()
-
         procs = []
 
         if proc_smac is not None:
@@ -554,26 +567,43 @@ def run_ensemble_builder(self,
                              'size 0.')
             return None
 
+    def refit(self, X, y):
+        if self._keep_models is not True:
+            raise ValueError(
+                "Predict can only be called if 'keep_models==True'")
+        if self.models_ is None or len(self.models_) == 0 or \
+                self.ensemble_ is None:
+            self._load_models()
+
+        for identifier in self.models_:
+            if identifier in self.ensemble_.get_model_identifiers():
+                model = self.models_[identifier]
+                # this updates the model inplace, it can then later be used in
+                # predict method
+                model.fit(X.copy(), y.copy())
+
+        self._can_predict = True
+
     def predict(self, X):
+        return np.argmax(self.predict_proba(X), axis=1)
+
+    def predict_proba(self, X):
         if self._keep_models is not True:
             raise ValueError(
                 "Predict can only be called if 'keep_models==True'")
-        if self._resampling_strategy not in  ['holdout',
-                                              'holdout-iterative-fit']:
+        if not self._can_predict and \
+                self._resampling_strategy not in  \
+                        ['holdout', 'holdout-iterative-fit']:
             raise NotImplementedError(
                 'Predict is currently only implemented for resampling '
                 'strategy holdout.')
 
-        if self.models_ is None or len(self.models_) == 0 or len(
-                self.ensemble_indices_) == 0:
+        if self.models_ is None or len(self.models_) == 0 or \
+                self.ensemble_ is None:
             self._load_models()
 
-        predictions = []
-        for identifier in self.models_:
-            if identifier not in self.ensemble_indices_:
-                continue
-
-            weight = self.ensemble_indices_[identifier]
+        all_predictions = []
+        for identifier in self.ensemble_.get_model_identifiers():
             model = self.models_[identifier]
 
             X_ = X.copy()
@@ -588,16 +618,16 @@ def predict(self, X):
                                      "while X_.shape is %s" %
                                      (model, str(prediction.shape),
                                       str(X_.shape)))
-            predictions.append(prediction * weight)
+            all_predictions.append(prediction)
 
-        if len(predictions) == 0:
+        if len(all_predictions) == 0:
             raise ValueError('Something went wrong generating the predictions. '
                              'The ensemble should consist of the following '
                              'models: %s, the following models were loaded: '
                              '%s' % (str(list(self.ensemble_indices_.keys())),
                                      str(list(self.models_.keys()))))
 
-        predictions = np.sum(np.array(predictions), axis=0)
+        predictions = self.ensemble_.predict(all_predictions)
         return predictions
 
     def _load_models(self):
@@ -610,42 +640,23 @@ def _load_models(self):
         if len(self.models_) == 0:
             raise ValueError('No models fitted!')
 
-        self.ensemble_indices_ = self._backend.load_ensemble_indices_weights(
-            seed)
+        self.ensemble_ = self._backend.load_ensemble(seed)
 
     def score(self, X, y):
         # fix: Consider only index 1 of second dimension
         # Don't know if the reshaping should be done there or in calculate_score
-        prediction = self.predict(X)
-        if self._task == BINARY_CLASSIFICATION:
-            prediction = prediction[:, 1].reshape((-1, 1))
+        prediction = self.predict_proba(X)
         return calculate_score(y, prediction, self._task,
                                self._metric, self._label_num,
                                logger=self._logger)
 
     def show_models(self):
-        if self.models_ is None or len(self.models_) == 0 or len(
-                self.ensemble_indices_) == 0:
-            self._load_models()
 
-        output = []
-        sio = six.StringIO()
-        for identifier in self.models_:
-            if identifier not in self.ensemble_indices_:
-                continue
-
-            weight = self.ensemble_indices_[identifier]
-            model = self.models_[identifier]
-            output.append((weight, model))
-
-        output.sort(reverse=True)
-
-        sio.write("[")
-        for weight, model in output:
-            sio.write("(%f, %s),\n" % (weight, model))
-        sio.write("]")
+        if self.models_ is None or len(self.models_) == 0 or \
+                self.ensemble_ is None:
+            self._load_models()
 
-        return sio.getvalue()
+        return self.ensemble_.pprint_ensemble_string(self.models_)
 
     def _save_ensemble_data(self, X, y):
         """Split dataset and store Data for the ensemble script.

autosklearn/cli/HPOlib_interface.py

@@ -82,7 +82,7 @@ def parse_cli():
     return args, parameters
 
 
-def parse_args(dataset, mode, seed, params, fold, folds):
+def parse_args(dataset, mode, seed, params, fold, folds, output_dir=None):
     if seed is None:
         seed = 1
 
@@ -107,10 +107,11 @@ def parse_args(dataset, mode, seed, params, fold, folds):
         mode_args = None
     else:
         raise ValueError(mode)
-    base_interface.main(dataset, mode, seed, params, mode_args=mode_args)
+    base_interface.main(dataset, mode, seed, params, mode_args=mode_args,
+                        output_dir=output_dir)
 
 
-def main():
+def main(output_dir=None):
     args, params = parse_cli()
     assert 'dataset' in args
     assert 'mode' in args
@@ -124,6 +125,7 @@ def main():
                params,
                int(args['fold']),
                int(args['folds']),
+               output_dir=output_dir
         )
 
 

autosklearn/cli/SMAC_interface.py

@@ -3,7 +3,8 @@
 
 from autosklearn.cli import base_interface
 
-def main():
+
+def main(output_dir=None):
     instance_name = sys.argv[1]
     instance_specific_information = sys.argv[2]
     cutoff_time = float(sys.argv[3])
@@ -45,7 +46,7 @@ def main():
         raise ValueError(mode)
 
     base_interface.main(instance_specific_information, mode,
-                        seed, params, mode_args=mode_args)
+                        seed, params, mode_args=mode_args, output_dir=output_dir)
 
 
 if __name__ == '__main__':