Merge branch 'development' of ssh://github.com/automl/auto-sklearn into development

Author: mfeurer

autosklearn/estimators.py

@@ -53,15 +53,7 @@ def refit(self, X, y):
     def fit_ensemble(self, y, task=None, metric=None, precision='32',
                      dataset_name=None, ensemble_nbest=None,
                      ensemble_size=None):
-        """Build the ensemble.
 
-        This method only needs to be called in the parallel mode.
-
-        Returns
-        -------
-        self
-
-        """
         return self._automl.fit_ensemble(y, task, metric, precision,
                                          dataset_name, ensemble_nbest,
                                          ensemble_size)
@@ -225,6 +217,16 @@ def __init__(self,
               an ensemble.
             * ``'model'`` : do not save any model files
 
+        configuration_mode : ``SMAC`` or ``ROAR``
+            Defines the configuration mode as described in the paper
+            `Sequential Model-Based Optimization for General Algorithm
+            Configuration <http://aad.informatik.uni-freiburg.de/papers/11-LION5-SMAC.pdf>`_:
+
+            * ``SMAC`` (default): Sequential Model-based Algorithm
+              Configuration, which is a Bayesian optimization algorithm
+            * ``ROAR``: Random Online Aggressive Racing, which is basically
+              random search
+
         Attributes
         ----------
 
@@ -305,6 +307,46 @@ def fit(self, *args, **kwargs):
     def fit_ensemble(self, y, task=None, metric=None, precision='32',
                      dataset_name=None, ensemble_nbest=None,
                      ensemble_size=None):
+        """Fit an ensemble to models trained during an optimization process.
+
+        All parameters are ``None`` by default. If no other value is given,
+        the default values which were set in a call to ``fit()`` are used.
+
+        Parameters
+        ----------
+        y : array-like
+            Target values.
+
+        task : int
+            A constant from the module ``autosklearn.constants``. Determines
+            the task type (binary classification, multiclass classification,
+            multilabel classification or regression).
+
+        metric : callable, optional (default='acc_metric')
+            An instance of :class:`autosklearn.metrics.Scorer` as created by
+            :meth:`autosklearn.metrics.make_scorer`. These are the `Built-in
+            Metrics`_.
+
+        precision : str
+            Numeric precision used when loading ensemble data. Can be either
+            ``'16'``, ``'32'`` or ``'64'``.
+
+        dataset_name : str
+            Name of the current data set.
+
+        ensemble_nbest : int
+            Determines how many models should be considered from the ensemble
+            building. This is inspired by a concept called library pruning
+            introduced in `Getting Most out of Ensemble Selection`.
+
+        ensemble_size : int
+            Size of the ensemble built by `Ensomble Selection`.
+
+        Returns
+        -------
+        self
+
+        """
         if self._automl is None:
             self._automl = self.build_automl()
         return self._automl.fit_ensemble(y, task, metric, precision,
@@ -338,12 +380,17 @@ def fit(self, X, y,
             The target classes.
 
         metric : callable, optional (default='acc_metric')
-            An instance of ``autosklearn.metrics.Scorer``.
+            An instance of :class:`autosklearn.metrics.Scorer` as created by
+            :meth:`autosklearn.metrics.make_scorer`. These are the `Built-in
+            Metrics`_.
 
         feat_type : list, optional (default=None)
             List of str of `len(X.shape[1])` describing the attribute type.
             Possible types are `Categorical` and `Numerical`. `Categorical`
-            attributes will be automatically One-Hot encoded.
+            attributes will be automatically One-Hot encoded. The values
+            used for a categorical attribute must be integers, obtainde for
+            example by `sklearn.preprocessing.LabelEncoder
+            <http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.LabelEncoder.html>`_.
 
         dataset_name : str, optional (default=None)
             Create nicer output. If None, a string will be determined by the

autosklearn/metrics/__init__.py

@@ -137,12 +137,10 @@ def make_scorer(name, score_func, greater_is_better=True, needs_proba=False,
                 needs_threshold=False, **kwargs):
     """Make a scorer from a performance metric or loss function.
 
-    Factory inspired by scikit-learn which wraps scoring functions to be used in
-    auto-sklearn. In difference to scikit-learn, auto-sklearn always needs to
-    call ``predict_proba`` in order to have predictions on a seperate validation
-    set to build ensembles with.
+    Factory inspired by scikit-learn which wraps scikit-learn scoring functions
+    to be used in auto-sklearn.
 
-    Paramaters
+    Parameters
     ----------
     score_func : callable
         Score function (or loss function) with signature

doc/api.rst

@@ -5,23 +5,93 @@
 APIs
 ****
 
+============
 Main modules
 ============
 
+~~~~~~~~~~~~~~
 Classification
 ~~~~~~~~~~~~~~
 
 .. autoclass:: autosklearn.classification.AutoSklearnClassifier
     :members:
     :inherited-members: show_models, fit_ensemble, refit
 
+~~~~~~~~~~
 Regression
 ~~~~~~~~~~
 
 .. autoclass:: autosklearn.regression.AutoSklearnRegressor
     :members:
     :inherited-members: show_models, fit_ensemble, refit
 
+=======
+Metrics
+=======
+
+.. autofunction:: autosklearn.metrics.make_scorer
+
+~~~~~~~~~~~~~~~~
+Built-in Metrics
+~~~~~~~~~~~~~~~~
+
+Classification
+~~~~~~~~~~~~~~
+
+.. autoclass:: autosklearn.metrics.accuracy
+
+.. autoclass:: autosklearn.metrics.balanced_accuracy
+
+.. autoclass:: autosklearn.metrics.f1
+
+.. autoclass:: autosklearn.metrics.f1_macro
+
+.. autoclass:: autosklearn.metrics.f1_micro
+
+.. autoclass:: autosklearn.metrics.f1_samples
+
+.. autoclass:: autosklearn.metrics.f1_weighted
+
+.. autoclass:: autosklearn.metrics.roc_auc
+
+.. autoclass:: autosklearn.metrics.precision
+
+.. autoclass:: autosklearn.metrics.precision_macro
+
+.. autoclass:: autosklearn.metrics.precision_micro
+
+.. autoclass:: autosklearn.metrics.precision_samples
+
+.. autoclass:: autosklearn.metrics.precision_weighted
+
+.. autoclass:: autosklearn.metrics.average_precision
+
+.. autoclass:: autosklearn.metrics.recall
+
+.. autoclass:: autosklearn.metrics.recall_macro
+
+.. autoclass:: autosklearn.metrics.recall_micro
+
+.. autoclass:: autosklearn.metrics.recall_samples
+
+.. autoclass:: autosklearn.metrics.recall_weighted
+
+.. autoclass:: autosklearn.metrics.log_loss
+
+.. autoclass:: autosklearn.metrics.pac_score
+
+Regression
+~~~~~~~~~~
+
+.. autoclass:: autosklearn.metrics.r2
+
+.. autoclass:: autosklearn.metrics.mean_squared_error
+
+.. autoclass:: autosklearn.metrics.mean_absolute_error
+
+.. autoclass:: autosklearn.metrics.median_absolute_error
+
+====================
 Extension Interfaces
 ====================
 

doc/conf.py

@@ -131,9 +131,11 @@
     # be in the form [(name, page), ..]
     'navbar_links': [
         ('Start', 'index'),
+        ('Releases', 'releases'),
+        ('Installation', 'installation'),
+        ('Manual', 'manual'),
         ('API', 'api'),
         ('Extending', 'extending'),
-        ('Manual', 'manual'),
     ],
 
     # Render the next and previous page links in navbar. (Default: true)

doc/extending.rst

@@ -8,7 +8,7 @@ Extending auto-sklearn
 
 auto-sklearn can be easily extended with new classification, regression and
 feature preprocessing methods. In order to do so, a user has to implement a
-wrapper class and make it known to auto-sklearn. This manual will walk you
+wrapper class and register it to auto-sklearn. This manual will walk you
 through the process.
 
 

doc/index.rst

@@ -30,9 +30,7 @@ Example
     >>> import sklearn.model_selection
     >>> import sklearn.datasets
     >>> import sklearn.metrics
-    >>> digits = sklearn.datasets.load_digits()
-    >>> X = digits.data
-    >>> y = digits.target
+    >>> X, y = sklearn.datasets.load_digits(return_X_y=True)
     >>> X_train, X_test, y_train, y_test = \
             sklearn.model_selection.train_test_split(X, y, random_state=1)
     >>> automl = autosklearn.classification.AutoSklearnClassifier()
@@ -44,34 +42,12 @@ Example
 This will run for one hour should result in an accuracy above 0.98.
 
 
-Installation
-************
-**Prerequisities**: *auto-sklearn* is written in python and was developed
-with Ubuntu. It should run on other Linux distributions, but won't work on a MAC
-or on a windows PC. We aim to always support the two latests python versions,
-which are 3.4 and 3.5 at the moment. It is built around scikit-learn 0.17.1 and
-needs a compiler for C++ 11.
-
-Please install all dependencies manually with:
-
-.. code:: bash
-
-    curl https://raw.githubusercontent.com/automl/auto-sklearn/master/requirements.txt | xargs -n 1 -L 1 pip install
-
-Then install *auto-sklearn*
-
-.. code:: bash
-
-    pip install auto-sklearn
-
-We recommend installing *auto-sklearn* into a `virtual environment
-<http://docs.python-guide.org/en/latest/dev/virtualenvs/>`_.
-
 Manual
 ******
 
-* :ref:`API`
+* :ref:`installation`
 * :ref:`manual`
+* :ref:`API`
 * :ref:`extending`
 
 
@@ -108,7 +84,7 @@ references to the following paper:
 Contributing
 ************
 
-We appreciate all contribution to auto-sklearn, from bug reports,
+We appreciate all contribution to auto-sklearn, from bug reports and
 documentation to new features. If you want to contribute to the code, you can
 pick an issue from the `issue tracker <https://github.com/automl/auto-sklearn/issues>`_
 which is marked with `Needs contributer`.
@@ -121,24 +97,5 @@ which is marked with `Needs contributer`.
     .com/automl/auto-sklearn/issues>`_ before starting to work.
 
 When developing new features, please create a new branch from the development
-branch. Prior to submitting a pull request, make sure that all tests are
+branch. When to submitting a pull request, make sure that all tests are
 still passing.
-
-Contributors
-************
-
-* Matthias Feurer
-* Katharina Eggensperger
-* Jost Tobias Springenberg
-* Aaron Klein
-* Anatolii Domashnev
-* Alexander Sapronov
-* Stefan Falkner
-* Manuel Blum
-* Hector Mendoza
-* Farooq Ahmed Zuberi
-* Frank Hutter
-* Diego Kobylkin
-* Marius Lindauer
-
-