Merge branch 'dev' into doc_readme

Author: vm-aifluence-jro

qolmat/imputations/imputers.py

@@ -1,5 +1,6 @@
 import warnings
 from typing import Dict, List, Optional, Union
+from abc import abstractmethod
 
 import numpy as np
 import pandas as pd
@@ -41,19 +42,38 @@ class Imputer(_BaseImputer):
 
     def __init__(
         self,
-        groups: List[str] = [],
         columnwise: bool = False,
         shrink: bool = False,
-        hyperparams: Dict = {},
         random_state: Union[None, int, np.random.RandomState] = None,
+        missing_values=np.nan,
+        groups: List[str] = [],
+        hyperparams: Dict = {},
     ):
-        self.hyperparams_user = hyperparams
-        self.groups = groups
         self.columnwise = columnwise
         self.shrink = shrink
         self.random_state = random_state
-
-    def fit_transform(self, df: pd.DataFrame) -> pd.DataFrame:
+        self.missing_values = missing_values
+        self.groups = groups
+        self.hyperparams = hyperparams
+
+    def _more_tags(self):
+        """Define tags for scikit-learn"""
+
+        return {
+            "allow_nan": True,
+            "requires_fit": False,
+            "_xfail_checks": {
+                "check_parameters_default_constructible": "The imputer need Dict as a parammeter",
+                "check_no_attributes_set_in_init": """The imputer can define an attribute
+                modifiable in init""",
+            },
+        }
+
+    def fit(self, X, y: pd.DataFrame = None):
+        X = self._validate_data(X, force_all_finite="allow-nan")
+        return self
+
+    def fit_transform(self, X: pd.DataFrame, y=None) -> pd.DataFrame:
         """
         Returns a dataframe with same shape as `df`, unchanged values, where all nans are replaced
         by non-nan values.
@@ -70,24 +90,27 @@ def fit_transform(self, df: pd.DataFrame) -> pd.DataFrame:
         pd.DataFrame
             Imputed dataframe.
         """
-        if not isinstance(df, pd.DataFrame):
-            raise ValueError("Input has to be a pandas.DataFrame.")
+        self.fit(X)
+
+        if not isinstance(X, (pd.DataFrame, np.ndarray)):
+            raise ValueError("Input has to be a pandas.DataFrame or numpy.ndarray.")
+        df = pd.DataFrame(X)
         for column in df:
             if df[column].isnull().all():
                 raise ValueError("Input contains a column full of NaN")
         self.rng = sku.check_random_state(self.random_state)
         if hasattr(self, "estimator") and hasattr(self.estimator, "random_state"):
             self.estimator.random_state = self.rng
 
-        hyperparams = self.hyperparams_user.copy()
+        hyperparams = self.hyperparams.copy()
         if hasattr(self, "hyperparams_optim"):
             hyperparams.update(self.hyperparams_optim)
         cols_with_nans = df.columns[df.isna().any()]
 
         if self.groups == []:
-            self.ngroups = pd.Series(0, index=df.index).rename("_ngroup")
+            self.ngroups_ = pd.Series(0, index=df.index).rename("_ngroup")
         else:
-            self.ngroups = df.groupby(self.groups).ngroup().rename("_ngroup")
+            self.ngroups_ = df.groupby(self.groups).ngroup().rename("_ngroup")
 
         if self.columnwise:
             df_imputed = df.copy()
@@ -156,8 +179,8 @@ def impute_element(self, df: pd.DataFrame) -> pd.DataFrame:
             raise ValueError("Input has to be a pandas.DataFrame.")
         df = df.copy()
         if self.groups:
-            # groupby = utils.custom_groupby(df, self.groups)
-            groupby = df.groupby(self.ngroups, group_keys=False)
+            # groupby = utils.custom_groupby(df, groups)
+            groupby = df.groupby(self.ngroups_, group_keys=False)
             if self.shrink:
                 imputation_values = groupby.transform(self.fit_transform_element)
             else:
@@ -173,6 +196,10 @@ def impute_element(self, df: pd.DataFrame) -> pd.DataFrame:
 
         return df
 
+    @abstractmethod
+    def fit_transform_element(self, df: pd.DataFrame):
+        return df
+
 
 class ImputerOracle(Imputer):
     """
@@ -195,7 +222,7 @@ def __init__(
         super().__init__()
         self.df = df
 
-    def fit_transform(self, df: pd.DataFrame) -> pd.DataFrame:
+    def fit_transform(self, X: pd.DataFrame, y=None) -> pd.DataFrame:
         """Impute df with corresponding known values
 
         Parameters
@@ -207,8 +234,10 @@ def fit_transform(self, df: pd.DataFrame) -> pd.DataFrame:
         pd.DataFrame
             dataframe imputed with premasked values
         """
-        if not isinstance(df, pd.DataFrame):
-            raise ValueError("Input has to be a pandas.DataFrame.")
+        self.fit(X)
+        if not isinstance(X, (pd.DataFrame, np.ndarray)):
+            raise ValueError("Input has to be a pandas.DataFrame or numpy.ndarray.")
+        df = pd.DataFrame(X)
         return df.fillna(self.df)
 
 
@@ -244,7 +273,12 @@ def __init__(
         groups: List[str] = [],
     ) -> None:
         super().__init__(groups=groups, columnwise=True, shrink=True)
-        self.fit_transform_element = pd.DataFrame.mean
+
+    def _more_tags(self):
+        return {"allow_nan": True, "requires_fit": False}
+
+    def fit_transform_element(self, df: pd.DataFrame):
+        return pd.DataFrame.mean(df)
 
 
 class ImputerMedian(Imputer):
@@ -279,7 +313,9 @@ def __init__(
         groups: List[str] = [],
     ) -> None:
         super().__init__(groups=groups, columnwise=True, shrink=True)
-        self.fit_transform_element = pd.DataFrame.median
+
+    def fit_transform_element(self, df: pd.DataFrame):
+        return pd.DataFrame.median(df)
 
 
 class ImputerMode(Imputer):
@@ -314,7 +350,9 @@ def __init__(
         groups: List[str] = [],
     ) -> None:
         super().__init__(groups=groups, columnwise=True, shrink=True)
-        self.fit_transform_element = lambda df: df.mode().iloc[0]
+
+    def fit_transform_element(self, df: pd.DataFrame):
+        return df.mode().iloc[0]
 
 
 class ImputerShuffle(Imputer):
@@ -647,6 +685,7 @@ def __init__(
         super().__init__(groups=groups, columnwise=False, hyperparams=hyperparams)
         self.n_neighbors = n_neighbors
         self.weights = weights
+        self.hyperparams_optim: Dict = {}
 
     def fit_transform_element(self, df: pd.DataFrame) -> pd.DataFrame:
         imputer = KNNImputer(
@@ -663,7 +702,8 @@ class ImputerMICE(Imputer):
     This class implements an iterative imputer in the multivariate case.
     It imputes each Series within a DataFrame multiple times using an iteration of fits
     and transformations to reach a stable state of imputation each time.
-    It uses sklearn.impute.IterativeImputer, see the docs for more information about the arguments.
+    It uses sklearn.impute.IterativeImputer, see the docs for more information about the
+    arguments.
 
     Parameters
     ----------
@@ -711,6 +751,7 @@ def __init__(
             random_state=random_state,
         )
         self.estimator = estimator
+        self.hyperparams_optim: Dict = {}
 
     def fit_transform_element(self, df: pd.DataFrame) -> pd.DataFrame:
         iterative_imputer = IterativeImputer(estimator=self.estimator, **self.hyperparams_element)
@@ -769,6 +810,7 @@ def __init__(
         self.columnwise = False
         self.estimator = estimator
         self.handler_nan = handler_nan
+        self.hyperparams_optim: Dict = {}
 
     def get_params_fit(self) -> Dict:
         return {}
@@ -842,8 +884,8 @@ class ImputerRPCA(Imputer):
     """
     This class implements the Robust Principal Component Analysis imputation.
 
-    The imputation minimizes a loss function combining a low-rank criterium on the dataframe and a
-    L1 penalization on the residuals.
+    The imputation minimizes a loss function combining a low-rank criterium on the dataframe and
+    a L1 penalization on the residuals.
 
     Parameters
     ----------
@@ -852,10 +894,11 @@ class ImputerRPCA(Imputer):
     method : str
         Name of the RPCA method:
             "PCP" for basic RPCA, bad at imputing
-            "noisy" for noisy RPCA, with possible regularisations, wihch is recommended since it is
-            more stable
+            "noisy" for noisy RPCA, with possible regularisations, wihch is recommended since
+            it is more stable
     columnwise : bool
-        For the RPCA method to be applied columnwise (with reshaping of each column into an array)
+        For the RPCA method to be applied columnwise (with reshaping of
+        each column into an array)
         or to be applied directly on the dataframe. By default, the value is set to False.
     """
 
@@ -875,6 +918,7 @@ def __init__(
         )
 
         self.method = method
+        self.hyperparams_optim: Dict = {}
 
     def fit_transform_element(self, df: pd.DataFrame) -> pd.DataFrame:
         if not isinstance(df, pd.DataFrame):
@@ -890,7 +934,7 @@ def fit_transform_element(self, df: pd.DataFrame) -> pd.DataFrame:
         X = df.values.T
         M, A = model.decompose_rpca_signal(X)
         df_imputed = pd.DataFrame((M + A).T, index=df.index, columns=df.columns)
-        df_imputed = df.where(df.isna(), df_imputed)
+        df_imputed = df.where(~df.isna(), df_imputed)
 
         return df_imputed
 
@@ -933,6 +977,7 @@ def __init__(
             random_state=random_state,
         )
         self.model = model
+        self.hyperparams_optim: Dict = {}
 
     def fit_transform_element(self, df: pd.DataFrame) -> pd.DataFrame:
         if self.model == "multinormal":

tests/benchmark/test_comparator.py

@@ -35,7 +35,8 @@
 index_tuples_expected = pd.MultiIndex.from_product(
     [["mae", "wmape", "KL_columnwise"], ["col1", "col2"]]
 )
-data_expected = [3.0, 0.5, 0.75, 0.5, 37.88948, 39.68123]
+# data_expected = [3.0, 0.5, 0.75, 0.5, 37.88948, 39.68123]
+data_expected = [4.467175, 7.467187, 1.116794, 7.467187, 37.491336, 36.977574]
 result_expected = pd.Series(data_expected, index=index_tuples_expected)
 
 
@@ -70,4 +71,4 @@ def test_benchmark_comparator_compare(df1: pd.DataFrame, imputer: str) -> None:
     else:
         result = comparison.compare(df_origin)
         result_expected_DataFrame = pd.DataFrame(result_expected)
-        np.testing.assert_allclose(result, result_expected_DataFrame, atol=1e-5)
+        np.testing.assert_allclose(result, result_expected_DataFrame, atol=1e-3)

tests/imputations/test_imputers.py

@@ -4,6 +4,7 @@
 import pandas as pd
 import pytest
 from sklearn.ensemble import ExtraTreesRegressor
+from sklearn.utils.estimator_checks import parametrize_with_checks
 
 from qolmat.imputations import imputers
 
@@ -145,7 +146,7 @@ def test_ImputerInterpolation_fit_transform(df: pd.DataFrame) -> None:
 
 @pytest.mark.parametrize("df", [df_timeseries])
 def test_ImputerResiduals_fit_transform(df: pd.DataFrame) -> None:
-    imputer = imputers.ImputerResiduals(7)
+    imputer = imputers.ImputerResiduals(period=7)
     result = imputer.fit_transform(df)
     expected = pd.DataFrame(
         {
@@ -191,7 +192,7 @@ def test_ImputerMICE_fit_transform(df: pd.DataFrame) -> None:
 
 @pytest.mark.parametrize("df", [df_incomplete])
 def test_ImputerRegressor_fit_transform(df: pd.DataFrame) -> None:
-    imputer = imputers.ImputerRegressor(model=ExtraTreesRegressor())
+    imputer = imputers.ImputerRegressor(estimator=ExtraTreesRegressor())
     result = imputer.fit_transform(df)
     expected = pd.DataFrame(
         {
@@ -209,7 +210,7 @@ def test_ImputerRPCA_fit_transform(df: pd.DataFrame) -> None:
     expected = pd.DataFrame(
         {
             "col1": [i for i in range(20)],
-            "col2": [0, 10.5, 2, 10.5, 2] + [i for i in range(5, 20)],
+            "col2": [0, 25.375562, 2, 29.396932, 2] + [i for i in range(5, 20)],
         }
     )
     np.testing.assert_allclose(result, expected)
@@ -229,3 +230,27 @@ def test_ImputerEM_fit_transform(df: pd.DataFrame) -> None:
         }
     )
     np.testing.assert_allclose(result, expected, atol=1e-6)
+
+
+@parametrize_with_checks(
+    [
+        imputers.Imputer(),
+        imputers.ImputerOracle(df_complete),
+        imputers.ImputerMean(),
+        imputers.ImputerMedian(),
+        imputers.ImputerMode(),
+        imputers.ImputerShuffle(),
+        imputers.ImputerLOCF(),
+        imputers.ImputerNOCB(),
+        imputers.ImputerInterpolation(),
+        imputers.ImputerResiduals(period=7),
+        imputers.KNNImputer(),
+        imputers.ImputerMICE(),
+        imputers.ImputerRegressor(),
+        imputers.ImputerRPCA(),
+        imputers.ImputerEM(),
+    ]
+)
+def test_sklearn_compatible_estimator(estimator: imputers.Imputer, check: Any) -> None:
+    """Check compatibility with sklearn, using sklearn estimator checks API."""
+    check(estimator)