Merge pull request #391 from PaulWestenthanner/fix/issue_313

simplified polynomial wrapper, added tests, fixed bugs

Author: PaulWestenthanner

category_encoders/wrapper.py

@@ -4,7 +4,7 @@
 from sklearn.model_selection import StratifiedKFold
 import category_encoders as encoders
 import pandas as pd
-from typing import Dict
+from typing import Dict, Optional
 
 
 class PolynomialWrapper(BaseEstimator, TransformerMixin):
@@ -70,28 +70,32 @@ class PolynomialWrapper(BaseEstimator, TransformerMixin):
     None
     """
 
-    def __init__(self, feature_encoder):
-        self.feature_encoder = feature_encoder
+    def __init__(self, feature_encoder: utils.BaseEncoder):
+        self.feature_encoder: utils.BaseEncoder = feature_encoder
         self.feature_encoders: Dict[str, utils.BaseEncoder] = {}
-        self.label_encoder = None
+        self.label_encoder: Optional[encoders.OneHotEncoder] = None
 
     def fit(self, X, y, **kwargs):
         # unite the input into pandas types
         X, y = utils.convert_inputs(X, y)
         y = pd.DataFrame(y, columns=['target'])
 
         # apply one-hot-encoder on the label
-        self.label_encoder = encoders.OneHotEncoder(handle_missing='error', handle_unknown='error', cols=['target'], drop_invariant=True,
+        self.label_encoder = encoders.OneHotEncoder(handle_missing='error',
+                                                    handle_unknown='error',
+                                                    cols=['target'],
+                                                    drop_invariant=True,
                                                     use_cat_names=True)
         labels = self.label_encoder.fit_transform(y)
         labels.columns = [column[7:] for column in labels.columns]
         labels = labels.iloc[:, 1:]  # drop one label
 
-        # train the feature encoders
+        # train the feature encoders, it is important to reset feature encoders first
+        self.feature_encoders = {}
         for class_name, label in labels.items():
             self.feature_encoders[class_name] = copy.deepcopy(self.feature_encoder).fit(X, label)
 
-    def transform(self, X):
+    def transform(self, X, y=None):
         # unite the input into pandas types
         X = utils.convert_input(X)
 
@@ -101,8 +105,14 @@ def transform(self, X):
         all_new_features = pd.DataFrame()
 
         # transform the features
+        if y is not None:
+            y = self.label_encoder.transform(pd.DataFrame({"target": y}))
         for class_name, feature_encoder in self.feature_encoders.items():
-            encoded = feature_encoder.transform(X)
+            if y is not None:
+                y_transform = y[f"target_{class_name}"]
+            else:
+                y_transform = None
+            encoded = feature_encoder.transform(X, y_transform)
 
             # decorate the encoded features with the label class suffix
             new_features = encoded[feature_encoder.cols]
@@ -117,42 +127,8 @@ def transform(self, X):
         return result
 
     def fit_transform(self, X, y=None, **fit_params):
-        # When we are training the feature encoders, we have to use fit_transform() method on the features.
-
-        # unite the input into pandas types
-        X, y = utils.convert_inputs(X, y)
-        y = y.to_frame()
-        y.columns = ["target"]
-
-        # apply one-hot-encoder on the label
-        self.label_encoder = encoders.OneHotEncoder(handle_missing='error', handle_unknown='error', cols=['target'], drop_invariant=True,
-                                                    use_cat_names=True)
-        labels = self.label_encoder.fit_transform(y)
-        labels.columns = [column[7:] for column in labels.columns]
-        labels = labels.iloc[:, 1:]  # drop one label
-
-        # initialization of the feature encoders
-        encoded = None
-        feature_encoder = None
-        all_new_features = pd.DataFrame()
-
-        # fit_transform the feature encoders
-        for class_name, label in labels.items():
-            feature_encoder = copy.deepcopy(self.feature_encoder)
-            encoded = feature_encoder.fit_transform(X, label)
-
-            # decorate the encoded features with the label class suffix
-            new_features = encoded[feature_encoder.cols]
-            new_features.columns = [str(column) + '_' + class_name for column in new_features.columns]
-
-            all_new_features = pd.concat((all_new_features, new_features), axis=1)
-            self.feature_encoders[class_name] = feature_encoder
-
-        # add features that were not encoded
-        result = pd.concat((encoded[encoded.columns[~encoded.columns.isin(feature_encoder.cols)]],
-                            all_new_features), axis=1)
-
-        return result
+        self.fit(X, y, **fit_params)
+        return self.transform(X, y)
 
 
 class NestedCVWrapper(BaseEstimator, TransformerMixin):

tests/test_wrapper.py

@@ -51,20 +51,37 @@ def test_transform_only_selected(self):
 
         # combination fit() + transform()
         wrapper.fit(x, y)
-        result = wrapper.transform(x)
-        print(result)
+        result = wrapper.transform(x, y)
         self.assertEqual(len(result.columns), 4, 'We expect 2 untouched features + f2 target encoded into 2 features')
 
         # directly fit_transform()
         wrapper = PolynomialWrapper(encoders.LeaveOneOutEncoder(cols=['f2']))
         result2 = wrapper.fit_transform(x, y)
-        print(result2)
         self.assertEqual(len(result2.columns), 4, 'We expect 2 untouched features + f2 target encoded into 2 features')
 
-        # in the case of leave-one-out, we expect different results, because leave-one-out principle
-        # is applied only on the training data (to decrease overfitting) while the testing data
-        # use the whole statistics (to be as accurate as possible).
-        self.assertFalse(result.iloc[0, 3] == result2.iloc[0, 3])
+        pd.testing.assert_frame_equal(result, result2)
+
+    def test_refit_stateless(self):
+        # test that when the encoder is fitted multiple times no old state is carried
+        x = pd.DataFrame([
+            ['a', 'b', 'c'],
+            ['a', 'b', 'c'],
+            ['b', 'b', 'c'],
+            ['b', 'b', 'b'],
+            ['b', 'b', 'b'],
+            ['a', 'b', 'a'],
+        ], columns=['f1', 'f2', 'f3'])
+        y1 = ['bee', 'cat', 'dog', 'dog', 'dog', 'dog']
+        y2 = ['bee', 'cat', 'duck', 'duck', 'duck', 'duck']
+        wrapper = PolynomialWrapper(encoders.TargetEncoder())
+        result_first_fit = wrapper.fit_transform(x, y1)
+        expected_categories_1 = {"cat", "dog"}  # 'bee' is dropped since first label is always dropped
+        expected_categories_2 = {"cat", "duck"}
+        self.assertEqual(set(wrapper.label_encoder.category_mapping[0]["mapping"].index), {"bee", "cat", "dog"})
+        self.assertEqual(set(wrapper.feature_encoders.keys()), expected_categories_1)
+        result_second_fit = wrapper.fit_transform(x, y2)
+        self.assertEqual(set(wrapper.label_encoder.category_mapping[0]["mapping"].index), {"bee", "cat", "duck"})
+        self.assertEqual(set(wrapper.feature_encoders.keys()), expected_categories_2)
 
 
 class TestNestedCVWrapper(TestCase):