Implementation of both use_clones and fit_base_estimators (#670)

* implementation of both use_clones and fit_base_estimators for EnsembleVoteClassifier, StackingClassifier and StackingCVClassifier as well as tests for these to parameters

* some formatting changes, e.g., sort imports

* updated CHANGELOG and user_guide

* travis mvn fix

* add conftest

* mod conftest

* mod conftest

* mod conftest

* mod conftest

* upd travis

* fix test case

* minor updates

Co-authored-by: Katrina Ni <[email protected]>
Co-authored-by: rasbt <[email protected]>

Author: 3 people

ci/.travis_test.sh

@@ -44,4 +44,4 @@ if [[ "$NOTEBOOKS" == "true" ]]; then
       find sources -name "*.ipynb" -not -path "sources/user_guide/image/*" -exec jupyter nbconvert --to notebook --execute {} \;
 
     fi
-fi
+fi

docs/sources/CHANGELOG.md

@@ -23,7 +23,7 @@ The CHANGELOG for the current development version is available at
 
 ##### Changes
 
-- -
+- Implemented both `use_clones` and `fit_base_estimators` (previously `refit` in `EnsembleVoteClassifier`) for `EnsembleVoteClassifier` and `StackingClassifier`. ([#670](https://github.com/rasbt/mlxtend/pull/670) via [Katrina Ni](https://github.com/nilichen))
 
 ##### Bug Fixes
 

docs/sources/user_guide/classifier/EnsembleVoteClassifier.ipynb

@@ -8,15 +8,15 @@
 #
 # License: BSD 3 clause
 
-from sklearn.base import BaseEstimator
-from sklearn.base import ClassifierMixin
-from sklearn.base import TransformerMixin
-from sklearn.preprocessing import LabelEncoder
-from sklearn.base import clone
+import numpy as np
+import warnings
+from sklearn.base import (BaseEstimator, ClassifierMixin, TransformerMixin,
+                          clone)
 from sklearn.exceptions import NotFittedError
-from ..externals.name_estimators import _name_estimators
+from sklearn.preprocessing import LabelEncoder
+
 from ..externals import six
-import numpy as np
+from ..externals.name_estimators import _name_estimators
 
 
 class EnsembleVoteClassifier(BaseEstimator, ClassifierMixin, TransformerMixin):
@@ -28,9 +28,10 @@ class EnsembleVoteClassifier(BaseEstimator, ClassifierMixin, TransformerMixin):
     clfs : array-like, shape = [n_classifiers]
         A list of classifiers.
         Invoking the `fit` method on the `VotingClassifier` will fit clones
-        of those original classifiers that will
+        of those original classifiers
         be stored in the class attribute
-        `self.clfs_` if `refit=True` (default).
+        if `use_clones=True` (default) and
+        `fit_base_estimators=True` (default).
     voting : str, {'hard', 'soft'} (default='hard')
         If 'hard', uses predicted class labels for majority rule voting.
         Else if 'soft', predicts the class label based on the argmax of
@@ -47,22 +48,33 @@ class EnsembleVoteClassifier(BaseEstimator, ClassifierMixin, TransformerMixin):
         - `verbose=2`: Prints info about the parameters of the clf being fitted
         - `verbose>2`: Changes `verbose` param of the underlying clf to
            self.verbose - 2
-    refit : bool (default: True)
+    use_clones : bool (default: True)
+        Clones the classifiers for stacking classification if True (default)
+        or else uses the original ones, which will be refitted on the dataset
+        upon calling the `fit` method. Hence, if use_clones=True, the original
+        input classifiers will remain unmodified upon using the
+        StackingClassifier's `fit` method.
+        Setting `use_clones=False` is
+        recommended if you are working with estimators that are supporting
+        the scikit-learn fit/predict API interface but are not compatible
+        to scikit-learn's `clone` function.
+    fit_base_estimators : bool (default: True)
         Refits classifiers in `clfs` if True; uses references to the `clfs`,
         otherwise (assumes that the classifiers were already fit).
-        Note: refit=False is incompatible to mist scikit-learn wrappers!
+        Note: fit_base_estimators=False will enforce use_clones to be False,
+        and is incompatible to most scikit-learn wrappers!
         For instance, if any form of cross-validation is performed
         this would require the re-fitting classifiers to training folds, which
-        would raise a NotFitterError if refit=False.
+        would raise a NotFitterError if fit_base_estimators=False.
         (New in mlxtend v0.6.)
 
     Attributes
     ----------
     classes_ : array-like, shape = [n_predictions]
     clf : array-like, shape = [n_predictions]
-        The unmodified input classifiers
+        The input classifiers; may be overwritten if `use_clones=False`
     clf_ : array-like, shape = [n_predictions]
-        Fitted clones of the input classifiers
+        Fitted input classifiers; clones if `use_clones=True`
 
     Examples
     --------
@@ -96,15 +108,19 @@ class EnsembleVoteClassifier(BaseEstimator, ClassifierMixin, TransformerMixin):
     For more usage examples, please see
     http://rasbt.github.io/mlxtend/user_guide/classifier/EnsembleVoteClassifier/
     """
+
     def __init__(self, clfs, voting='hard',
-                 weights=None, verbose=0, refit=True):
+                 weights=None, verbose=0,
+                 use_clones=True,
+                 fit_base_estimators=True):
 
         self.clfs = clfs
         self.named_clfs = {key: value for key, value in _name_estimators(clfs)}
         self.voting = voting
         self.weights = weights
         self.verbose = verbose
-        self.refit = refit
+        self.use_clones = use_clones
+        self.fit_base_estimators = fit_base_estimators
 
     def fit(self, X, y, sample_weight=None):
         """Learn weight coefficients from training data for each classifier.
@@ -146,12 +162,17 @@ def fit(self, X, y, sample_weight=None):
         self.le_.fit(y)
         self.classes_ = self.le_.classes_
 
-        if not self.refit:
-            self.clfs_ = [clf for clf in self.clfs]
+        if not self.fit_base_estimators:
+            warnings.warn("fit_base_estimators=False "
+                          "enforces use_clones to be `False`")
+            self.use_clones = False
 
+        if self.use_clones:
+            self.clfs_ = clone(self.clfs)
         else:
-            self.clfs_ = [clone(clf) for clf in self.clfs]
+            self.clfs_ = self.clfs
 
+        if self.fit_base_estimators:
             if self.verbose > 0:
                 print("Fitting %d classifiers..." % (len(self.clfs)))
 
@@ -204,8 +225,8 @@ def predict(self, X):
             predictions = self._predict(X)
 
             maj = np.apply_along_axis(lambda x:
-                                      np.argmax(np.bincount(x,
-                                                weights=self.weights)),
+                                      np.argmax(np.bincount(
+                                          x, weights=self.weights)),
                                       axis=1,
                                       arr=predictions)
 
@@ -266,15 +287,15 @@ def get_params(self, deep=True):
                 for key, value in six.iteritems(step.get_params(deep=True)):
                     out['%s__%s' % (name, key)] = value
 
-            for key, value in six.iteritems(super(EnsembleVoteClassifier,
-                                            self).get_params(deep=False)):
+            for key, value in six.iteritems(
+                    super(EnsembleVoteClassifier, self).get_params(deep=False)):
                 out['%s' % key] = value
             return out
 
     def _predict(self, X):
         """Collect results from clf.predict calls."""
 
-        if self.refit:
+        if self.fit_base_estimators:
             return np.asarray([clf.predict(X) for clf in self.clfs_]).T
         else:
             return np.asarray([self.le_.transform(clf.predict(X))

docs/sources/user_guide/classifier/StackingClassifier.ipynb

@@ -8,14 +8,15 @@
 #
 # License: BSD 3 clause
 
+import numpy as np
+import warnings
+from scipy import sparse
+from sklearn.base import TransformerMixin, clone
+
 from ..externals.estimator_checks import check_is_fitted
 from ..externals.name_estimators import _name_estimators
 from ..utils.base_compostion import _BaseXComposition
 from ._base_classification import _BaseStackingClassifier
-from scipy import sparse
-from sklearn.base import TransformerMixin
-from sklearn.base import clone
-import numpy as np
 
 
 class StackingClassifier(_BaseXComposition, _BaseStackingClassifier,
@@ -30,7 +31,8 @@ class StackingClassifier(_BaseXComposition, _BaseStackingClassifier,
         Invoking the `fit` method on the `StackingClassifer` will fit clones
         of these original classifiers that will
         be stored in the class attribute
-        `self.clfs_`.
+        `self.clfs_` if `use_clones=True` (default) and
+        `fit_base_estimators=True` (default).
     meta_classifier : object
         The meta-classifier to be fitted on the ensemble of
         classifiers
@@ -77,6 +79,16 @@ class StackingClassifier(_BaseXComposition, _BaseStackingClassifier,
         recommended if you are working with estimators that are supporting
         the scikit-learn fit/predict API interface but are not compatible
         to scikit-learn's `clone` function.
+    fit_base_estimators: bool (default: True)
+        Refits classifiers in `classifiers` if True; uses references to the
+        `classifiers`, otherwise (assumes that the classifiers were
+        already fit).
+        Note: fit_base_estimators=False will enforce use_clones to be False,
+        and is incompatible to most scikit-learn wrappers!
+        For instance, if any form of cross-validation is performed
+        this would require the re-fitting classifiers to training folds, which
+        would raise a NotFitterError if fit_base_estimators=False.
+        (New in mlxtend v0.6.)
 
     Attributes
     ----------
@@ -100,7 +112,7 @@ def __init__(self, classifiers, meta_classifier,
                  average_probas=False, verbose=0,
                  use_features_in_secondary=False,
                  store_train_meta_features=False,
-                 use_clones=True):
+                 use_clones=True, fit_base_estimators=True):
 
         self.classifiers = classifiers
         self.meta_classifier = meta_classifier
@@ -117,6 +129,7 @@ def __init__(self, classifiers, meta_classifier,
         self.use_features_in_secondary = use_features_in_secondary
         self.store_train_meta_features = store_train_meta_features
         self.use_clones = use_clones
+        self.fit_base_estimators = fit_base_estimators
 
     @property
     def named_classifiers(self):
@@ -143,33 +156,39 @@ def fit(self, X, y, sample_weight=None):
         self : object
 
         """
+        if not self.fit_base_estimators:
+            warnings.warn("fit_base_estimators=False "
+                          "enforces use_clones to be `False`")
+            self.use_clones = False
+
         if self.use_clones:
             self.clfs_ = clone(self.classifiers)
             self.meta_clf_ = clone(self.meta_classifier)
         else:
             self.clfs_ = self.classifiers
             self.meta_clf_ = self.meta_classifier
 
-        if self.verbose > 0:
-            print("Fitting %d classifiers..." % (len(self.classifiers)))
+        if self.fit_base_estimators:
+            if self.verbose > 0:
+                print("Fitting %d classifiers..." % (len(self.classifiers)))
 
-        for clf in self.clfs_:
+            for clf in self.clfs_:
 
-            if self.verbose > 0:
-                i = self.clfs_.index(clf) + 1
-                print("Fitting classifier%d: %s (%d/%d)" %
-                      (i, _name_estimators((clf,))[0][0], i, len(self.clfs_)))
-
-            if self.verbose > 2:
-                if hasattr(clf, 'verbose'):
-                    clf.set_params(verbose=self.verbose - 2)
-
-            if self.verbose > 1:
-                print(_name_estimators((clf,))[0][1])
-            if sample_weight is None:
-                clf.fit(X, y)
-            else:
-                clf.fit(X, y, sample_weight=sample_weight)
+                if self.verbose > 0:
+                    i = self.clfs_.index(clf) + 1
+                    print("Fitting classifier%d: %s (%d/%d)" %
+                          (i, _name_estimators((clf,))[0][0], i, len(self.clfs_)))
+
+                if self.verbose > 2:
+                    if hasattr(clf, 'verbose'):
+                        clf.set_params(verbose=self.verbose - 2)
+
+                if self.verbose > 1:
+                    print(_name_estimators((clf,))[0][1])
+                if sample_weight is None:
+                    clf.fit(X, y)
+                else:
+                    clf.fit(X, y, sample_weight=sample_weight)
 
         meta_features = self.predict_meta_features(X)
 

mlxtend/classifier/ensemble_vote.py

@@ -9,16 +9,17 @@
 #
 # License: BSD 3 clause
 
-from ..externals.name_estimators import _name_estimators
-from ..externals.estimator_checks import check_is_fitted
-from ..utils.base_compostion import _BaseXComposition
-from ._base_classification import _BaseStackingClassifier
 import numpy as np
 from scipy import sparse
-from sklearn.base import TransformerMixin
-from sklearn.base import clone
+from sklearn.base import TransformerMixin, clone
 from sklearn.model_selection import cross_val_predict
 from sklearn.model_selection._split import check_cv
+
+from ..externals.estimator_checks import check_is_fitted
+from ..externals.name_estimators import _name_estimators
+from ..utils.base_compostion import _BaseXComposition
+from ._base_classification import _BaseStackingClassifier
+
 # from sklearn.utils import check_X_y
 
 
@@ -35,7 +36,7 @@ class StackingCVClassifier(_BaseXComposition, _BaseStackingClassifier,
         A list of classifiers.
         Invoking the `fit` method on the `StackingCVClassifer` will fit clones
         of these original classifiers that will
-        be stored in the class attribute `self.clfs_`.
+        be stored in the class attribute `self.clfs_` if `use_clones=True`.
     meta_classifier : object
         The meta-classifier to be fitted on the ensemble of
         classifiers
@@ -139,6 +140,7 @@ class StackingCVClassifier(_BaseXComposition, _BaseStackingClassifier,
     http://rasbt.github.io/mlxtend/user_guide/classifier/StackingCVClassifier/
 
     """
+
     def __init__(self, classifiers, meta_classifier,
                  use_probas=False, drop_proba_col=None,
                  cv=2, shuffle=True,
@@ -245,10 +247,10 @@ def fit(self, X, y, groups=None, sample_weight=None):
                 print(_name_estimators((model,))[0][1])
 
             prediction = cross_val_predict(
-                    model, X, y, groups=groups, cv=final_cv,
-                    n_jobs=self.n_jobs, fit_params=fit_params,
-                    verbose=self.verbose, pre_dispatch=self.pre_dispatch,
-                    method='predict_proba' if self.use_probas else 'predict')
+                model, X, y, groups=groups, cv=final_cv,
+                n_jobs=self.n_jobs, fit_params=fit_params,
+                verbose=self.verbose, pre_dispatch=self.pre_dispatch,
+                method='predict_proba' if self.use_probas else 'predict')
 
             if not self.use_probas:
                 prediction = prediction[:, np.newaxis]