Merge pull request #397 from PaulWestenthanner/docs/gray_rankhot

Docs/gray rankhot

Author: PaulWestenthanner

CHANGELOG.md

@@ -1,5 +1,11 @@
 unreleased
 ==========
+
+* added: ignore option for one-hot-encoding
+* fixed: external dependency in unit test
+* fixed: gaps in ordinal encoding if nan values are present
+* fixed: sklearn complicance: add `feature_names_in_` attribute
+* fixed: add RankHotEncoder in documentation
 * fixed: return correct mapping in one hot encoder `category_mapping` property (issue #256)
 
 v2.6.0

category_encoders/ordinal.py

@@ -220,7 +220,12 @@ def ordinal_encoding(X_in, mapping=None, cols=None, handle_unknown='value', hand
             mapping_out = []
             for col in cols:
                 nan_identity = np.nan
-                categories = list(X[col].unique())
+                categories = X[col].unique()
+                # make nan last category
+                if pd.isna(categories).any():
+                    categories = [c for c in categories if not pd.isna(c)] + [nan_identity]
+                else:
+                    categories = categories.tolist()
                 if util.is_category(X[col].dtype):
                     # Avoid using pandas category dtype meta-data if possible, see #235, #238.
                     if X[col].dtype.ordered:

category_encoders/rankhot.py

@@ -226,7 +226,7 @@ def generate_mapping(self):
             index = []
             new_columns = []
 
-            for cat_name, class_ in values.iteritems():
+            for cat_name, class_ in values.items():
                 if self.use_cat_names:
                     n_col_name = f"{col}_{cat_name}"
                     found_count = found_column_counts.get(n_col_name, 0)

category_encoders/utils.py

@@ -80,7 +80,9 @@ def convert_inputs(X, y, columns=None, index=None, deep=False):
         # N.B.: If either was already pandas, it keeps its index.
 
         if any(X.index != y.index):
-            raise ValueError("`X` and `y` both have indexes, but they do not match.")
+            msg = "`X` and `y` both have indexes, but they do not match. If you are shuffling your input data on " \
+                  "purpose (e.g. via permutation_test_score) use np arrays instead of data frames / series"
+            raise ValueError(msg)
         if X.shape[0] != y.shape[0]:
             raise ValueError("The length of X is " + str(X.shape[0]) + " but length of y is " + str(y.shape[0]) + ".")
     return X, y

docs/source/gray.rst

@@ -1,6 +1,6 @@
 Gray
 ====
 
-.. autoclass:: category_encoders.gray.GaryEncoder
+.. autoclass:: category_encoders.gray.GrayEncoder
     :members:
     :inherited-members:

docs/source/index.rst

@@ -52,11 +52,12 @@ To use:
     encoder = ce.MEstimateEncoder(cols=[...])
     encoder = ce.OneHotEncoder(cols=[...])
     encoder = ce.OrdinalEncoder(cols=[...])
-    encoder = ce.SumEncoder(cols=[...])
     encoder = ce.PolynomialEncoder(cols=[...])
+    encoder = ce.QuantileEncoder(cols=[...])
+    encoder = ce.RankHotEncoder(cols=[...])
+    encoder = ce.SumEncoder(cols=[...])
     encoder = ce.TargetEncoder(cols=[...])
     encoder = ce.WOEEncoder(cols=[...])
-    encoder = ce.QuantileEncoder(cols=[...])
 
     encoder.fit(X, y)
     X_cleaned = encoder.transform(X_dirty)
@@ -85,6 +86,7 @@ Contents:
    ordinal
    polynomial
    quantile
+   rankhot
    sum
    summary
    targetencoder

docs/source/rankhot.rst

@@ -0,0 +1,6 @@
+RankHotEncoder
+==============
+
+.. autoclass:: category_encoders.rankhot.RankHotEncoder
+    :members:
+    :inherited-members:

tests/test_encoders.py

@@ -439,20 +439,14 @@ def test_duplicate_index_value(self):
                 self.assertEqual(5, len(result))
 
     def test_string_index(self):
-        # https://github.com/scikit-learn-contrib/categorical-encoding/issues/131
-
-        bunch = sklearn.datasets.fetch_openml(name="house_prices", as_frame=True)
-        y = (bunch.target > 200000).values
-        X = pd.DataFrame(bunch.data, columns=bunch.feature_names)
-        X.index = X.index.values.astype(str)
-
-        display_cols = ["Id", "MSSubClass", "MSZoning", "YearBuilt", "Heating", "CentralAir"]
-        X = X[display_cols]
+        train = pd.DataFrame({'city': ['chicago', 'denver']})
+        target = [0, 1]
+        train.index = train.index.values.astype(str)
 
         for encoder_name in encoders.__all__:
             with self.subTest(encoder_name=encoder_name):
-                enc = getattr(encoders, encoder_name)(cols=['CentralAir', 'Heating'])
-                result = enc.fit_transform(X, y)
+                enc = getattr(encoders, encoder_name)()
+                result = enc.fit_transform(train, target)
                 self.assertFalse(result.isnull().values.any(), 'There should not be any missing value!')
 
     def test_get_feature_names_out(self):
@@ -609,8 +603,7 @@ def test_metamorphic(self):
         x3 = pd.DataFrame(data={'x': ['A', 'B', 'B']})  # DataFrame
         x4 = pd.Series(['A', 'B', 'B'], dtype='category')  # Series with category data type
         x5 = np.array(['A', 'B', 'B'])  # Numpy
-        x6 = [np.NaN, 'B', 'B']  # Missing value
-        x7 = ['Z', 'Y', 'Y']  # Different strings, reversed alphabetic ordering (it works because we look at the order of appearance, not at alphabetic order)
+        x6 = ['Z', 'Y', 'Y']  # Different strings, reversed alphabetic ordering (it works because we look at the order of appearance, not at alphabetic order)
 
         y = [1, 1, 0]
 
@@ -636,19 +629,13 @@ def test_metamorphic(self):
                 result5 = enc5.fit_transform(x5, y)
                 self.assertTrue((result1.values == result5.values).all())
 
-                # gray encoder and rankhot re-orders inputs so that nan is last, hence the output is changed
+                # gray encoder actually does re-order inputs
+                # rankhot encoder respects order, in this example the order is switched
                 if encoder_name not in ["GrayEncoder", "RankHotEncoder"]:
                     enc6 = getattr(encoders, encoder_name)()
                     result6 = enc6.fit_transform(x6, y)
                     self.assertTrue((result1.values == result6.values).all())
 
-                # gray encoder actually does re-order inputs
-                # rankhot encoder respects order, in this example the order is switched
-                if encoder_name not in ["GrayEncoder", "RankHotEncoder"]:
-                    enc7 = getattr(encoders, encoder_name)()
-                    result7 = enc7.fit_transform(x7, y)
-                    self.assertTrue((result1.values == result7.values).all())
-
                 # Arguments
                 enc9 = getattr(encoders, encoder_name)(return_df=False)
                 result9 = enc9.fit_transform(x1, y)

tests/test_one_hot.py

@@ -159,31 +159,38 @@ def test_HandleMissingReturnNan(self):
             result,
             pd.DataFrame({'x_A': [1, np.nan, 0], 'x_B': [0, np.nan, 1]}),
         )
-        
+
     def test_HandleMissingIgnore(self):
         train = pd.DataFrame({'x': ['A', 'B', np.nan],
                               'y': ['A', None, 'A'],
                               'z': [np.NaN, 'B', 'B']})
         train['z'] = train['z'].astype('category')
-        
+
         expected_result = pd.DataFrame({'x_A': [1, 0, 0],
                                         'x_B': [0, 1, 0],
                                         'y_A': [1, 0, 1],
                                         'z_B': [0, 1, 1]})    
         encoder = encoders.OneHotEncoder(handle_missing='ignore', use_cat_names=True)
         result = encoder.fit_transform(train)
-        
+
         pd.testing.assert_frame_equal(result, expected_result)
-        
+
     def test_HandleMissingIgnore_ExpectMappingUsed(self):
-        train = pd.DataFrame({'city': ['Chicago', np.NaN,'Geneva']})
+        train = pd.DataFrame({'city': ['Chicago', np.NaN, 'Geneva']})
         expected_result = pd.DataFrame({'city_1': [1, 0, 0],
-                                        'city_3': [0, 0, 1]})
+                                        'city_2': [0, 0, 1]})
 
         encoder = encoders.OneHotEncoder(handle_missing='ignore')
         result = encoder.fit(train).transform(train)
+        expected_mapping = pd.DataFrame([
+            [1, 0],
+            [0, 1],
+            [0, 0],
+            [0, 0],
+        ], columns=["city_1", "city_2"], index=[1, 2, -2, -1])
 
         pd.testing.assert_frame_equal(expected_result, result)
+        pd.testing.assert_frame_equal(expected_mapping, encoder.category_mapping[0]["mapping"])
 
     def test_HandleMissingIndicator_NanInTrain_ExpectAsColumn(self):
         train = ['A', 'B', np.nan]
@@ -271,7 +278,7 @@ def test_inverse_transform_BothFieldsAreReturnNanWithNan_ExpectValueError(self):
         enc = encoders.OneHotEncoder(handle_missing='return_nan', handle_unknown='return_nan')
         enc.fit(train)
         result = enc.transform(test)
-        
+
         message = 'inverse_transform is not supported because transform impute '\
                   'the unknown category nan when encode city'
 

tests/test_ordinal.py

@@ -177,6 +177,18 @@ def test_HaveNaNInTrain_ExpectCodedAsOne(self):
 
         self.assertEqual(expected, result)
 
+    def test_NoGaps(self):
+        train = pd.DataFrame({"city": ["New York", np.nan, "Rio", None, "Rosenheim"]})
+        expected_mapping_value = pd.Series([1, 2, 3, 4], index=["New York", "Rio", "Rosenheim", np.nan])
+        expected_mapping_return_nan = pd.Series([1, 2, 3, -2], index=["New York", "Rio", "Rosenheim", np.nan])
+
+        enc_value = encoders.OrdinalEncoder(cols=["city"], handle_missing="value")
+        enc_value.fit(train)
+        pd.testing.assert_series_equal(expected_mapping_value, enc_value.mapping[0]["mapping"])
+        enc_return_nan = encoders.OrdinalEncoder(cols=["city"], handle_missing="return_nan")
+        enc_return_nan.fit(train)
+        pd.testing.assert_series_equal(expected_mapping_return_nan, enc_return_nan.mapping[0]["mapping"])
+
     def test_HaveNoneAndNan_ExpectCodesAsOne(self):
         train = pd.DataFrame({"city": [np.nan, None]})
         expected = [1, 1]