Merge pull request #25 from automl/development

Merge in latest developments

Author: mfeurer

.coveralls.yml

@@ -39,11 +39,12 @@ install:
 
   # Install requirements from other repos
   - pip install git+https://github.com/automl/HPOlibConfigSpace.git
-  - pip install git+https://github.com/automl/paramsklearn.git
+
   - python setup.py install
 
+# command to run tests, e.g. python setup.py test
 script:
   # - coverage run --source autosklearn setup.py test
-  - cd test && nosetests -v --with-coverage
+  - cd test && nosetests -v --with-coverage --cover-package=autosklearn
 
-after_success: coveralls
+after_success: coveralls

.travis.yml

@@ -0,0 +1,24 @@
+Copyright (c) 2014, Matthias Feurer
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of the <organization> nor the
+      names of its contributors may be used to endorse or promote products
+      derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

CHANGES.md

@@ -26,7 +26,7 @@
     convert_conf2smac_string
 from autosklearn.evaluation import calculate_score
 from autosklearn.util import StopWatch, get_logger, setup_logger, \
-    get_auto_seed, set_auto_seed, del_auto_seed, submit_process, paramsklearn, \
+    get_auto_seed, set_auto_seed, del_auto_seed, submit_process, pipeline, \
     Backend
 from autosklearn.util.smac import run_smac
 
@@ -76,7 +76,7 @@ def _create_search_space(tmp_dir, data_info, backend, watcher, logger,
     task_name = 'CreateConfigSpace'
     watcher.start_task(task_name)
     configspace_path = os.path.join(tmp_dir, 'space.pcs')
-    configuration_space = paramsklearn.get_configuration_space(
+    configuration_space = pipeline.get_configuration_space(
         data_info,
         include_estimators=include_estimators,
         include_preprocessors=include_preprocessors)
@@ -614,7 +614,11 @@ def _load_models(self):
             seed)
 
     def score(self, X, y):
+        # fix: Consider only index 1 of second dimension
+        # Don't know if the reshaping should be done there or in calculate_score
         prediction = self.predict(X)
+        if self._task == BINARY_CLASSIFICATION:
+            prediction = prediction[:, 1].reshape((-1, 1))
         return calculate_score(y, prediction, self._task,
                                self._metric, self._label_num,
                                logger=self._logger)
@@ -695,4 +699,4 @@ def _delete_output_directories(self):
                     pass
                 else:
                     print("Could not delete tmp dir: %s" %
-                          self._tmp_dir)
+                          self._tmp_dir)

LICENSE.txt

@@ -11,7 +11,7 @@
 from autosklearn.data.competition_data_manager import CompetitionDataManager
 from autosklearn.evaluation import CVEvaluator, HoldoutEvaluator, \
     NestedCVEvaluator, TestEvaluator, get_new_run_num
-from autosklearn.util.paramsklearn import get_configuration_space
+from autosklearn.util.pipeline import get_configuration_space
 from autosklearn.util import Backend
 
 

autosklearn/automl.py

@@ -32,8 +32,8 @@
 
 R2_METRIC = 10
 A_METRIC = 11
-REGRESSION_METRIC = [R2_METRIC, A_METRIC]
-METRIC = CLASSIFICATION_METRICS + REGRESSION_METRIC
+REGRESSION_METRICS = [R2_METRIC, A_METRIC]
+METRIC = CLASSIFICATION_METRICS + REGRESSION_METRICS
 STRING_TO_METRIC = {
     'acc': ACC_METRIC,
     'acc_metric': ACC_METRIC,

autosklearn/cli/base_interface.py

@@ -5,7 +5,7 @@
 import numpy as np
 import scipy.sparse
 
-from ParamSklearn.implementations.OneHotEncoder import OneHotEncoder
+from autosklearn.pipeline.implementations.OneHotEncoder import OneHotEncoder
 
 from autosklearn.util import predict_RAM_usage
 

autosklearn/constants.py

@@ -58,15 +58,20 @@ def get_predictions(dir_path, dir_path_list, include_num_runs,
         match = model_and_automl_re.search(model_name)
         automl_seed = int(match.group(1))
         num_run = int(match.group(2))
+
+        if model_name.endswith("/"):
+            model_name = model_name[:-1]
+        basename = os.path.basename(model_name)
+
         if (automl_seed, num_run) in include_num_runs:
             if precision == "16":
-                predictions = np.load(os.path.join(dir_path, model_name)).astype(dtype=np.float16)
+                predictions = np.load(os.path.join(dir_path, basename)).astype(dtype=np.float16)
             elif precision == "32":
-                predictions = np.load(os.path.join(dir_path, model_name)).astype(dtype=np.float32)
+                predictions = np.load(os.path.join(dir_path, basename)).astype(dtype=np.float32)
             elif precision == "64":
-                predictions = np.load(os.path.join(dir_path, model_name)).astype(dtype=np.float64)
+                predictions = np.load(os.path.join(dir_path, basename)).astype(dtype=np.float64)
             else:
-                predictions = np.load(os.path.join(dir_path, model_name))
+                predictions = np.load(os.path.join(dir_path, basename))
             result.append(predictions)
     return result
 
@@ -249,7 +254,10 @@ def main(autosklearn_tmp_dir,
         dir_ensemble_list_mtimes = []
 
         for dir_ensemble_file in dir_ensemble_list:
-            dir_ensemble_file = os.path.join(dir_ensemble, dir_ensemble_file)
+            if dir_ensemble_file.endswith("/"):
+                dir_ensemble_file = dir_ensemble_file[:-1]
+            basename = os.path.basename(dir_ensemble_file)
+            dir_ensemble_file = os.path.join(dir_ensemble, basename)
             mtime = os.path.getmtime(dir_ensemble_file)
             dir_ensemble_list_mtimes.append(mtime)
 
@@ -285,14 +293,18 @@ def main(autosklearn_tmp_dir,
 
         model_idx = 0
         for model_name in dir_ensemble_list:
+            if model_name.endswith("/"):
+                model_name = model_name[:-1]
+            basename = os.path.basename(model_name)
+
             if precision is "16":
-                predictions = np.load(os.path.join(dir_ensemble, model_name)).astype(dtype=np.float16)
+                predictions = np.load(os.path.join(dir_ensemble, basename)).astype(dtype=np.float16)
             elif precision is "32":
-                predictions = np.load(os.path.join(dir_ensemble, model_name)).astype(dtype=np.float32)
+                predictions = np.load(os.path.join(dir_ensemble, basename)).astype(dtype=np.float32)
             elif precision is "64":
-                predictions = np.load(os.path.join(dir_ensemble, model_name)).astype(dtype=np.float64)
+                predictions = np.load(os.path.join(dir_ensemble, basename)).astype(dtype=np.float64)
             else:
-                predictions = np.load(os.path.join(dir_ensemble, model_name))
+                predictions = np.load(os.path.join(dir_ensemble, basename))
             score = calculate_score(targets_ensemble, predictions,
                                     task_type, metric,
                                     predictions.shape[1])

autosklearn/data/abstract_data_manager.py

@@ -7,8 +7,8 @@
 
 import numpy as np
 import lockfile
-from ParamSklearn.classification import ParamSklearnClassifier
-from ParamSklearn.regression import ParamSklearnRegressor
+from autosklearn.pipeline.classification import SimpleClassificationPipeline
+from autosklearn.pipeline.regression import SimpleRegressionPipeline
 from sklearn.dummy import DummyClassifier, DummyRegressor
 
 from autosklearn.constants import *
@@ -106,13 +106,13 @@ def __init__(self, Datamanager, configuration=None,
             if self.configuration is None:
                 self.model_class = MyDummyRegressor
             else:
-                self.model_class = ParamSklearnRegressor
+                self.model_class = SimpleRegressionPipeline
             self.predict_function = self.predict_regression
         else:
             if self.configuration is None:
                 self.model_class = MyDummyClassifier
             else:
-                self.model_class = ParamSklearnClassifier
+                self.model_class = SimpleClassificationPipeline
             self.predict_function = self.predict_proba
 
         if num_run is None: