Speed up DropDuplicateFeatures (#614)

* Speed up DropDuplicateFeatures

* Extend the test case to test different data types

* Fix stylechecks issue

Author: Okroshiashvili

feature_engine/selection/drop_duplicate_features.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 from typing import List, Union
 
 import pandas as pd
@@ -99,7 +100,6 @@ def __init__(
         missing_values: str = "ignore",
         confirm_variables: bool = False,
     ):
-
         if missing_values not in ["raise", "ignore"]:
             raise ValueError("missing_values takes only values 'raise' or 'ignore'.")
 
@@ -136,42 +136,30 @@ def fit(self, X: pd.DataFrame, y: pd.Series = None):
             # check if dataset contains na
             _check_contains_na(X, self.variables_)
 
-        # create tuples of duplicated feature groups
-        self.duplicated_feature_sets_ = []
-
-        # set to collect features that are duplicated
-        self.features_to_drop_ = set()  # type: ignore
-
-        # create set of examined features
-        _examined_features = set()
-
-        for feature in self.variables_:
+        # collect duplicate features
+        _features_hashmap = defaultdict(list)
 
-            # append so we can remove when we create the combinations
-            _examined_features.add(feature)
+        # hash the features
+        _X_hash = pd.util.hash_pandas_object(X[self.variables_].T, index=False)
 
-            if feature not in self.features_to_drop_:
+        # group the features by hash
+        for feature, feature_hash in _X_hash.items():
+            _features_hashmap[feature_hash].append(feature)
 
-                _temp_set = set([feature])
-
-                # features that have not been examined, are not currently examined and
-                # were not found duplicates
-                _features_to_compare = [
-                    f
-                    for f in self.variables_
-                    if f not in _examined_features.union(self.features_to_drop_)
-                ]
-
-                # create combinations:
-                for f2 in _features_to_compare:
-
-                    if X[feature].equals(X[f2]):
-                        self.features_to_drop_.add(f2)
-                        _temp_set.add(f2)
+        # create tuples of duplicated feature groups
+        self.duplicated_feature_sets_ = [
+            set(duplicate)
+            for duplicate in _features_hashmap.values()
+            if len(duplicate) > 1
+        ]
 
-                # if there are duplicated features
-                if len(_temp_set) > 1:
-                    self.duplicated_feature_sets_.append(_temp_set)
+        # set to collect features that are duplicated
+        self.features_to_drop_ = {
+            item
+            for duplicates in _features_hashmap.values()
+            for item in duplicates[1:]
+            if duplicates and len(duplicates) > 1
+        }
 
         # save input features
         self._get_feature_names_in(X)

tests/test_selection/test_drop_duplicate_features.py

@@ -43,6 +43,23 @@ def df_duplicate_features_with_na():
     return df
 
 
+@pytest.fixture(scope="module")
+def df_duplicate_features_with_different_data_types():
+    data = {
+        "A": pd.Series([5.5] * 3).astype("float64"),
+        "B": 1,
+        "C": "foo",
+        "D": pd.Timestamp("20010102"),
+        "E": pd.Series([1.0] * 3).astype("float32"),
+        "F": False,
+        "G": pd.Series([1] * 3, dtype="int8"),
+    }
+
+    df = pd.DataFrame(data)
+
+    return df
+
+
 def test_drop_duplicates_features(df_duplicate_features):
     transformer = DropDuplicateFeatures()
     X = transformer.fit_transform(df_duplicate_features)
@@ -94,3 +111,18 @@ def test_with_df_with_na(df_duplicate_features_with_na):
         {"City", "City2"},
         {"Age", "Age2"},
     ]
+
+
+def test_with_different_data_types(df_duplicate_features_with_different_data_types):
+    transformer = DropDuplicateFeatures()
+    X = transformer.fit_transform(df_duplicate_features_with_different_data_types)
+    df = pd.DataFrame(
+        {
+            "A": pd.Series([5.5] * 3).astype("float64"),
+            "B": 1,
+            "C": "foo",
+            "D": pd.Timestamp("20010102"),
+            "F": False,
+        }
+    )
+    pd.testing.assert_frame_equal(X, df)