fixed and simplified count encoder

also fixed a bug in ordinal encoder

Author: PaulWestenthanner

category_encoders/count.py

@@ -4,15 +4,17 @@
 import numpy as np
 import pandas as pd
 import category_encoders.utils as util
+from category_encoders.ordinal import OrdinalEncoder
 
 from copy import copy
 from sklearn.base import BaseEstimator, TransformerMixin
 
 
 __author__ = 'joshua t. dunn'
 
-# COUNT_ENCODER BRANCH
+
 class CountEncoder(BaseEstimator, TransformerMixin):
+
     def __init__(self, verbose=0, cols=None, drop_invariant=False,
                  return_df=True, handle_unknown='value',
                  handle_missing='value',
@@ -118,6 +120,7 @@ def __init__(self, verbose=0, cols=None, drop_invariant=False,
         self.min_group_name = min_group_name
         self.combine_min_nan_groups = combine_min_nan_groups
         self.feature_names = None
+        self.ordinal_encoder = None
 
         self._check_set_create_attrs()
 
@@ -157,9 +160,17 @@ def fit(self, X, y=None, **kwargs):
         else:
             self.cols = util.convert_cols_to_list(self.cols)
 
+        self.ordinal_encoder = OrdinalEncoder(
+            verbose=self.verbose,
+            cols=self.cols,
+            handle_unknown='value',
+            handle_missing='value'
+        )
+        self.ordinal_encoder = self.ordinal_encoder.fit(X)
+        X_ordinal = self.ordinal_encoder.transform(X)
         self._check_set_create_dict_attrs()
 
-        self._fit_count_encode(X, y)
+        self._fit_count_encode(X_ordinal, y)
 
         X_temp = self.transform(X, override_return_df=True)
         self.feature_names = list(X_temp.columns)
@@ -236,28 +247,11 @@ def _fit_count_encode(self, X_in, y):
         self.mapping = {}
 
         for col in self.cols:
-            if X[col].isnull().any():
-                if self._handle_missing[col] == 'error':
-                    raise ValueError(
-                        'Missing data found in column %s at fit time.'
-                        % (col,)
-                    )
-
-                elif self._handle_missing[col] not in ['value', 'return_nan',  'error', None]:
-                    raise ValueError(
-                        '%s key in `handle_missing` should be one of: '
-                        ' `value`, `return_nan` and `error` not `%s`.'
-                        % (col, str(self._handle_missing[col]))
-                    )
-
-            self.mapping[col] = X[col].value_counts(
-                normalize=self._normalize[col],
-                dropna=False
-            )
-
-            self.mapping[col].index = self.mapping[col].index.astype(object)
-
-
+            mapping_values = X[col].value_counts(normalize=self._normalize[col])
+            ordinal_encoding = [m["mapping"] for m in self.ordinal_encoder.mapping if m["col"] == col][0]
+            reversed_ordinal_enc = {v: k for k, v in ordinal_encoding.to_dict().items()}
+            mapping_values.index = mapping_values.index.map(reversed_ordinal_enc)
+            self.mapping[col] = mapping_values
 
             if self._handle_missing[col] == 'return_nan':
                 self.mapping[col][np.NaN] = np.NaN
@@ -273,15 +267,15 @@ def _transform_count_encode(self, X_in, y):
         X = X_in.copy(deep=True)
 
         for col in self.cols:
-
-            X[col] = X.fillna(value=np.nan)[col]
+            # Treat None as np.nan
+            X[col] = pd.Series([el if el is not None else np.NaN for el in X[col]], index=X[col].index)
+            if self.handle_missing == "value":
+                if not util.is_category(X[col].dtype):
+                    X[col] = X[col].fillna(np.nan)
 
             if self._min_group_size is not None:
                 if col in self._min_group_categories.keys():
-                    X[col] = (
-                        X[col].map(self._min_group_categories[col])
-                        .fillna(X[col])
-                    )
+                    X[col] = X[col].map(self._min_group_categories[col]).fillna(X[col])
 
             X[col] = X[col].astype(object).map(self.mapping[col])
             if isinstance(self._handle_unknown[col], (int, np.integer)):

category_encoders/ordinal.py

@@ -290,15 +290,14 @@ def ordinal_encoding(X_in, mapping=None, cols=None, handle_unknown='value', hand
             for switch in mapping:
                 column = switch.get('col')
                 col_mapping = switch['mapping']
-                X[column] = X[column].map(col_mapping)
 
+                # Treat None as np.nan
+                X[column] = pd.Series([el if el is not None else np.NaN for el in X[column]], index=X[column].index)
+                X[column] = X[column].map(col_mapping)
                 if util.is_category(X[column].dtype):
-                    if not isinstance(col_mapping, pd.Series):
-                        col_mapping = pd.Series(col_mapping)
                     nan_identity = col_mapping.loc[col_mapping.index.isna()].values[0]
                     X[column] = X[column].cat.add_categories(nan_identity)
                     X[column] = X[column].fillna(nan_identity)
-
                 try:
                     X[column] = X[column].astype(int)
                 except ValueError as e:

tests/test_count.py

@@ -62,7 +62,7 @@ def test_count_handle_missing_string(self):
 
     def test_count_handle_missing_dict(self):
         """Test the handle_missing dict on 'none' and 'na_categorical'. 
-        We want to see differing behavour between 'none' and 'na_cat' cols."""
+        We want to see differing behaviour between 'none' and 'na_cat' cols."""
         enc = encoders.CountEncoder(
             handle_missing={'na_categorical': 'return_nan'}
         )
@@ -169,7 +169,7 @@ def test_count_combine_min_nan_groups_bool(self):
         self.assertTrue(pd.Series([9, 7, 4]).isin(out['na_categorical']).all())
         self.assertEqual(out['na_categorical'].unique().shape[0], 3)
         self.assertTrue(enc.mapping is not None)
-        self.assertIn(np.nan, enc.mapping['na_categorical'])
+        self.assertIn(np.NaN, enc.mapping['na_categorical'])
 
     def test_count_combine_min_nan_groups_dict(self):
         """Test the combine_min_nan_groups dict  on 'none' and 'na_categorical'."""

tests/test_ordinal.py

@@ -161,6 +161,16 @@ def test_HaveNoneAndNan_ExpectCodesAsOne(self):
 
         self.assertEqual(expected, result)
 
+        new_nan = pd.DataFrame({'city': [np.nan,]})
+        result_new_nan = enc.transform(new_nan)['city'].tolist()
+        expected_new_nan = [1]
+        self.assertEqual(expected_new_nan, result_new_nan)
+
+        new_none = pd.DataFrame({'city': [None, ]})
+        result_new_none = enc.transform(new_none)['city'].tolist()
+        expected_new_none = [1]
+        self.assertEqual(expected_new_none, result_new_none)
+
     def test_inverse_transform_HaveUnknown_ExpectWarning(self):
         train = pd.DataFrame({'city': ['chicago', 'st louis']})
         test = pd.DataFrame({'city': ['chicago', 'los angeles']})