Merge remote-tracking branch 'upstream/master' into fix-binary-encoder-for-columntransformer

Author: florian

category_encoders/binary.py

@@ -71,16 +71,9 @@ class BinaryEncoder(BaseEstimator, TransformerMixin):
 
     def __init__(self, verbose=0, cols=None, mapping=None, drop_invariant=False, return_df=True,
                  handle_unknown='value', handle_missing='value'):
-        self. verbose = verbose
-        self.cols = cols
-        self.mapping = mapping
-        self.drop_invariant = drop_invariant
-        self.return_df = return_df
-        self.handle_unknown = handle_unknown
-        self.handle_missing = handle_missing
-        self.base_n_encoder = ce.BaseNEncoder(base=2, verbose=self.verbose, cols=self.cols, mapping=self.mapping,
-                                              drop_invariant=self.drop_invariant, return_df=self.return_df,
-                                              handle_unknown=self.handle_unknown, handle_missing=self.handle_missing)
+        self.base_n_encoder = ce.BaseNEncoder(base=2, verbose=verbose, cols=cols, mapping=mapping,
+                                              drop_invariant=drop_invariant, return_df=return_df,
+                                              handle_unknown=handle_unknown, handle_missing=handle_missing)
 
     def fit(self, X, y=None, **kwargs):
         """Fit encoder according to X and y.

category_encoders/ordinal.py

@@ -144,7 +144,7 @@ def fit(self, X, y=None, **kwargs):
         self.mapping = categories
 
         X_temp = self.transform(X, override_return_df=True)
-        self.feature_names = X_temp.columns.values.tolist()
+        self.feature_names = X_temp.columns.tolist()
 
         # drop all output columns with 0 variance.
         if self.drop_invariant:

category_encoders/tests/helpers.py

@@ -0,0 +1,70 @@
+"""Helper functions that are used exclusively in the tests"""
+
+import numpy as np
+import random
+import pandas as pd
+import math
+
+
+def verify_numeric(X_test):
+    """
+    Test that all attributes in the DataFrame are numeric.
+    """
+    _NUMERIC_KINDS = set('buifc')
+    
+    for dt in X_test.dtypes:
+        assert(dt.kind in _NUMERIC_KINDS)
+
+def create_array(n_rows=1000, extras=False, has_none=True):
+    """
+    Creates a numpy dataset with some categorical variables.
+    """
+    ds = [[
+        random.random(),
+        random.random(),
+        random.choice(['A', 'B', 'C']),
+        random.choice(['A', 'B', 'C', 'D']) if extras else random.choice(['A', 'B', 'C']),
+        random.choice(['A', 'B', 'C', None, np.nan]) if has_none else random.choice(['A', 'B', 'C']),
+        random.choice(['A'])
+    ] for _ in range(n_rows)]
+
+    return np.array(ds)
+
+
+def create_dataset(n_rows=1000, extras=False, has_none=True):
+    """
+    Creates a dataset with some categorical variables.
+    """
+    random.seed(2001)
+    ds = [[
+        random.random(),                                                                        # Floats
+        random.choice([float('nan'), float('inf'), float('-inf'), -0, 0, 1, -1, math.pi]),      # Floats with edge scenarios
+        row,                                                                                    # Unique integers
+        str(row),                                                                               # Unique strings
+        random.choice(['A', 'B']) if extras else 'A',                                           # Invariant in the training data
+        random.choice(['A', 'B_b', 'C_c_c']),                                                   # Strings with underscores to test reverse_dummies()
+        random.choice(['A', 'B', 'C', None]) if has_none else random.choice(['A', 'B', 'C']),   # None
+        random.choice(['A', 'B', 'C', 'D']) if extras else random.choice(['A', 'B', 'C']),      # With a new string value
+        random.choice([12, 43, -32]),                                                           # Number in the column name
+        random.choice(['A', 'B', 'C']),                                                         # What is going to become the categorical column
+        random.choice(['A', 'B', 'C', np.nan])                                                  # Categorical with missing values
+    ] for row in range(n_rows)]
+
+    df = pd.DataFrame(ds, columns=['float', 'float_edge', 'unique_int', 'unique_str', 'invariant', 'underscore', 'none', 'extra', 321, 'categorical', 'categorical_na'])
+    df['categorical'] = pd.Categorical(df['categorical'], categories=['A', 'B', 'C'])
+    df['categorical_na'] = pd.Categorical(df['categorical_na'], categories=['A', 'B', 'C'])
+    return df
+
+
+def verify_inverse_transform(x, x_inv):
+    """
+    Verify x is equal to x_inv. The test returns true for NaN.equals(NaN) as it should.
+    """
+    assert x.equals(x_inv)
+
+
+def deep_round(A, ndigits=5):
+    """
+    Rounds numbers in a list of lists. Useful for approximate equality testing.
+    """
+    return [[round(val, ndigits) for val in sublst] for sublst in A]

category_encoders/tests/test_basen.py

@@ -111,14 +111,13 @@ def test_inverse_transform_BothFieldsAreReturnNanWithNan_ExpectValueError(self):
         enc = encoders.BaseNEncoder(handle_missing='return_nan', handle_unknown='return_nan')
         enc.fit(train)
         result = enc.transform(test)
+        
+        message = 'inverse_transform is not supported because transform impute '\
+                  'the unknown category nan when encode city'
 
-        with warnings.catch_warnings(record=True) as w:
+        with self.assertWarns(UserWarning, msg=message) as w:
             enc.inverse_transform(result)
 
-            self.assertEqual(1, len(w))
-            self.assertEqual('inverse_transform is not supported because transform impute '
-                             'the unknown category nan when encode city', str(w[0].message))
-
     def test_inverse_transform_HaveMissingAndNoUnknown_ExpectInversed(self):
         train = pd.DataFrame({'city': ['chicago', np.nan]})
         test = pd.DataFrame({'city': ['chicago', 'los angeles']})

category_encoders/tests/test_encoders.py

@@ -6,7 +6,7 @@
 import numpy as np
 import pandas as pd
 import sklearn
-import category_encoders.tests.test_helpers as th
+import category_encoders.tests.helpers as th
 from sklearn.utils.estimator_checks import check_transformer_general, check_transformers_unfitted
 from sklearn.compose import ColumnTransformer
 from unittest2 import TestSuite, TextTestRunner, TestCase  # or `from unittest import ...` if on Python 3.4+

category_encoders/tests/test_helpers.py

@@ -1,70 +1,42 @@
-"""Helper functions that are used exclusively in the tests"""
-
 import numpy as np
-import random
 import pandas as pd
-import math
-
+from unittest2 import TestCase  # or `from unittest import ...` if on Python 3.4+
 
-def verify_numeric(X_test):
-    """
-    Test that all attributes in the DataFrame are numeric.
-    """
-    for dt in X_test.dtypes:
-        numeric = False
-        if np.issubdtype(dt, np.dtype(int)) or np.issubdtype(dt, np.dtype(float)):
-            numeric = True
-        assert numeric
+from category_encoders.tests.helpers import verify_numeric
 
 
-def create_array(n_rows=1000, extras=False, has_none=True):
-    """
-    Creates a numpy dataset with some categorical variables.
-    """
-    ds = [[
-        random.random(),
-        random.random(),
-        random.choice(['A', 'B', 'C']),
-        random.choice(['A', 'B', 'C', 'D']) if extras else random.choice(['A', 'B', 'C']),
-        random.choice(['A', 'B', 'C', None, np.nan]) if has_none else random.choice(['A', 'B', 'C']),
-        random.choice(['A'])
-    ] for _ in range(n_rows)]
+class TestHelpers(TestCase):
 
-    return np.array(ds)
+    def test_is_numeric_pandas(self):
+        # Whole numbers, regardless of the byte length, should not raise AssertionError
+        X = pd.DataFrame(np.ones([5, 5]), dtype='int32')
+        verify_numeric(pd.DataFrame(X))
 
+        X = pd.DataFrame(np.ones([5, 5]), dtype='int64')
+        verify_numeric(pd.DataFrame(X))
 
-def create_dataset(n_rows=1000, extras=False, has_none=True):
-    """
-    Creates a dataset with some categorical variables.
-    """
-    random.seed(2001)
-    ds = [[
-        random.random(),                                                                        # Floats
-        random.choice([float('nan'), float('inf'), float('-inf'), -0, 0, 1, -1, math.pi]),      # Floats with edge scenarios
-        row,                                                                                    # Unique integers
-        str(row),                                                                               # Unique strings
-        random.choice(['A', 'B']) if extras else 'A',                                           # Invariant in the training data
-        random.choice(['A', 'B_b', 'C_c_c']),                                                   # Strings with underscores to test reverse_dummies()
-        random.choice(['A', 'B', 'C', None]) if has_none else random.choice(['A', 'B', 'C']),   # None
-        random.choice(['A', 'B', 'C', 'D']) if extras else random.choice(['A', 'B', 'C']),      # With a new string value
-        random.choice([12, 43, -32]),                                                           # Number in the column name
-        random.choice(['A', 'B', 'C']),                                                         # What is going to become the categorical column
-    ] for row in range(n_rows)]
+        # Strings should raise AssertionError
+        X = pd.DataFrame([['a', 'b', 'c'], ['d', 'e', 'f']])
+        with self.assertRaises(Exception):
+            verify_numeric(pd.DataFrame(X))
 
-    df = pd.DataFrame(ds, columns=['float', 'float_edge', 'unique_int', 'unique_str', 'invariant', 'underscore', 'none', 'extra', 321, 'categorical'])
-    df['categorical'] = pd.Categorical(df['categorical'], categories=['A', 'B', 'C'])
-    return df
+    def test_is_numeric_numpy(self):
+        # Whole numbers, regardless of the byte length, should not raise AssertionError
+        X = np.ones([5, 5], dtype='int32')
+        verify_numeric(pd.DataFrame(X))
 
+        X = np.ones([5, 5], dtype='int64')
+        verify_numeric(pd.DataFrame(X))
 
-def verify_inverse_transform(x, x_inv):
-    """
-    Verify x is equal to x_inv. The test returns true for NaN.equals(NaN) as it should.
-    """
-    assert x.equals(x_inv)
+        # Floats
+        X = np.ones([5, 5], dtype='float32')
+        verify_numeric(pd.DataFrame(X))
 
+        X = np.ones([5, 5], dtype='float64')
+        verify_numeric(pd.DataFrame(X))
 
-def deep_round(A, ndigits=5):
-    """
-    Rounds numbers in a list of lists. Useful for approximate equality testing.
-    """
-    return [[round(val, ndigits) for val in sublst] for sublst in A]
+    def test_verify_raises_AssertionError_on_categories(self):
+        # Categories should raise AssertionError
+        X = pd.DataFrame([['a', 'b', 'c'], ['d', 'e', 'f']], dtype='category')
+        with self.assertRaises(Exception):
+            verify_numeric(pd.DataFrame(X))

category_encoders/tests/test_leave_one_out.py

@@ -1,6 +1,6 @@
 import pandas as pd
 from unittest2 import TestCase  # or `from unittest import ...` if on Python 3.4+
-import category_encoders.tests.test_helpers as th
+import category_encoders.tests.helpers as th
 import numpy as np
 
 import category_encoders as encoders

category_encoders/tests/test_one_hot.py

@@ -1,8 +1,8 @@
 import pandas as pd
-from unittest import TestCase  # or `from unittest import ...` if on Python 3.4+
+from unittest2 import TestCase  # or `from unittest import ...` if on Python 3.4+
 import numpy as np
 import warnings
-import category_encoders.tests.test_helpers as th
+import category_encoders.tests.helpers as th
 
 import category_encoders as encoders
 
@@ -227,14 +227,13 @@ def test_inverse_transform_BothFieldsAreReturnNanWithNan_ExpectValueError(self):
         enc = encoders.OneHotEncoder(handle_missing='return_nan', handle_unknown='return_nan')
         enc.fit(train)
         result = enc.transform(test)
+        
+        message = 'inverse_transform is not supported because transform impute '\
+                  'the unknown category nan when encode city'
 
-        with warnings.catch_warnings(record=True) as w:
+        with self.assertWarns(UserWarning, msg=message) as w:
             enc.inverse_transform(result)
 
-            self.assertEqual(1, len(w))
-            self.assertEqual('inverse_transform is not supported because transform impute '
-                             'the unknown category nan when encode city', str(w[0].message))
-
     def test_inverse_transform_HaveMissingAndNoUnknown_ExpectInversed(self):
         train = pd.DataFrame({'city': ['chicago', np.nan]})
         test = pd.DataFrame({'city': ['chicago', 'los angeles']})

category_encoders/tests/test_ordinal.py

@@ -1,6 +1,6 @@
 import pandas as pd
 from unittest2 import TestCase  # or `from unittest import ...` if on Python 3.4+
-import category_encoders.tests.test_helpers as th
+import category_encoders.tests.helpers as th
 import numpy as np
 import warnings
 import category_encoders as encoders
@@ -35,7 +35,7 @@ def test_ordinal(self):
         self.assertIn(-1, set(out['extra'].values))
         self.assertTrue(len(enc.mapping) > 0)
 
-        enc = encoders.OrdinalEncoder(verbose=1, return_df=True, handle_unknown='return _nan')
+        enc = encoders.OrdinalEncoder(verbose=1, return_df=True, handle_unknown='return_nan')
         enc.fit(X)
         out = enc.transform(X_t)
         out_cats = [x for x in set(out['extra'].values) if np.isfinite(x)]
@@ -137,13 +137,12 @@ def test_inverse_transform_HaveUnknown_ExpectWarning(self):
         enc.fit(train)
         result = enc.transform(test)
 
-        with warnings.catch_warnings(record=True) as w:
+        message = 'inverse_transform is not supported because transform impute '\
+                             'the unknown category -1 when encode city'
+        
+        with self.assertWarns(UserWarning, msg=message) as w:
             enc.inverse_transform(result)
 
-            self.assertEqual(1, len(w))
-            self.assertEqual('inverse_transform is not supported because transform impute '
-                             'the unknown category -1 when encode city', str(w[0].message))
-
     def test_inverse_transform_HaveNanInTrainAndHandleMissingValue_ExpectReturnedWithNan(self):
         train = pd.DataFrame({'city': ['chicago', np.nan]})
 
@@ -169,14 +168,13 @@ def test_inverse_transform_BothFieldsAreReturnNanWithNan_ExpectValueError(self):
         enc = encoders.OrdinalEncoder(handle_missing='return_nan', handle_unknown='return_nan')
         enc.fit(train)
         result = enc.transform(test)
+        
+        message = 'inverse_transform is not supported because transform impute '\
+                             'the unknown category nan when encode city'
 
-        with warnings.catch_warnings(record=True) as w:
+        with self.assertWarns(UserWarning, msg=message) as w:
             enc.inverse_transform(result)
 
-            self.assertEqual(1, len(w))
-            self.assertEqual('inverse_transform is not supported because transform impute '
-                             'the unknown category nan when encode city', str(w[0].message))
-
     def test_inverse_transform_HaveMissingAndNoUnknown_ExpectInversed(self):
         train = pd.DataFrame({'city': ['chicago', np.nan]})
         test = pd.DataFrame({'city': ['chicago', 'los angeles']})

category_encoders/tests/test_target_encoder.py

@@ -1,6 +1,6 @@
 import pandas as pd
 from unittest2 import TestCase  # or `from unittest import ...` if on Python 3.4+
-import category_encoders.tests.test_helpers as th
+import category_encoders.tests.helpers as th
 import numpy as np
 
 import category_encoders as encoders