Merge pull request #65 from Quantmetry/review_dcor

grouped imputation tested, metric_optim argument added to comparator

Author: JulienRoussel77

qolmat/benchmark/comparator.py

@@ -34,6 +34,7 @@ def __init__(
         generator_holes: _HoleGenerator,
         metrics: List = ["mae", "wmape", "KL_columnwise"],
         dict_config_opti: Optional[Dict[str, Any]] = {},
+        metric_optim: str = "mse",
         max_evals: int = 10,
         verbose: bool = False,
     ):
@@ -42,6 +43,7 @@ def __init__(
         self.generator_holes = generator_holes
         self.metrics = metrics
         self.dict_config_opti = dict_config_opti
+        self.metric_optim = metric_optim
         self.max_evals = max_evals
         self.verbose = verbose
 
@@ -77,6 +79,7 @@ def evaluate_errors_sample(
         imputer: Any,
         df: pd.DataFrame,
         dict_config_opti_imputer: Dict[str, Any] = {},
+        metric_optim: str = "mse",
     ) -> pd.Series:
         """Evaluate the errors in the cross-validation
 
@@ -88,6 +91,8 @@ def evaluate_errors_sample(
             dataframe to impute
         dict_config_opti_imputer : Dict
             search space for tested_model's hyperparameters
+        metric_optim : str
+            Loss function used when imputers undergo hyperparameter optimization
 
         Returns
         -------
@@ -99,7 +104,6 @@ def evaluate_errors_sample(
         for df_mask in self.generator_holes.split(df_origin):
             df_corrupted = df_origin.copy()
             df_corrupted[df_mask] = np.nan
-            metric_optim = "mae"
             imputer_opti = hyperparameters.optimize(
                 imputer,
                 df,
@@ -142,7 +146,7 @@ def compare(
 
             try:
                 dict_errors[name] = self.evaluate_errors_sample(
-                    imputer, df, dict_config_opti_imputer
+                    imputer, df, dict_config_opti_imputer, self.metric_optim
                 )
                 print(f"Tested model: {type(imputer).__name__}")
             except Exception as excp:

qolmat/imputations/em_sampler.py

@@ -177,18 +177,22 @@ def fit(self, X: NDArray):
         mask_na = np.isnan(X)
 
         # first imputation
-        X_sample_last = utils.linear_interpolation(X)
-        self.fit_distribution(X_sample_last)
+        X = utils.linear_interpolation(X)
+        print("fit")
+        print(X)
+        print("fit_distribution")
+        self.fit_distribution(X)
+        print("...")
 
         for iter_em in range(self.max_iter_em):
-            X_sample_last = self._sample_ou(X_sample_last, mask_na)
+            X = self._sample_ou(X, mask_na)
 
             if self._check_convergence():
                 # print(f"EM converged after {iter_em} iterations.")
                 break
 
         self.dict_criteria_stop = {key: [] for key in self.dict_criteria_stop}
-        self.X_sample_last = X_sample_last
+        self.X_sample_last = X
         return self
 
     def transform(self, X: NDArray) -> NDArray:
@@ -314,7 +318,9 @@ def get_loglikelihood(self, X: NDArray) -> float:
         if np.all(np.isclose(self.cov, 0)):
             return 0
         else:
-            return scipy.stats.multivariate_normal.logpdf(X.T, self.means, self.cov).mean()
+            return scipy.stats.multivariate_normal.logpdf(
+                X.T, self.means, self.cov, allow_singular=True
+            ).mean()
 
     def _maximize_likelihood(self, X: NDArray, mask_na: NDArray, dt: float = np.nan) -> NDArray:
         """

qolmat/imputations/imputers.py

@@ -1223,10 +1223,8 @@ def fit(self, X: pd.DataFrame, y=None):
             Returns self.
         """
         super().fit(X)
-        df = self._check_input(X)
         hyperparameters = self.get_hyperparams()
         self.imputer_ = KNNImputer(metric="nan_euclidean", **hyperparameters)
-        self.imputer_.fit(df)
         return self
 
     def _transform_element(self, df: pd.DataFrame, col: str = "__all__") -> pd.DataFrame:
@@ -1252,7 +1250,7 @@ def _transform_element(self, df: pd.DataFrame, col: str = "__all__") -> pd.DataF
             Input has to be a pandas.DataFrame.
         """
         self._check_dataframe(df)
-        results = self.imputer_.transform(df)
+        results = self.imputer_.fit_transform(df)
         return pd.DataFrame(data=results, columns=df.columns, index=df.index)
 
 
@@ -1329,10 +1327,10 @@ def fit(self, X: pd.DataFrame, y=None):
         """
         hyperparams = self.get_hyperparams()
         super().fit(X)
-        df = self._check_input(X)
         self.imputer_ = IterativeImputer(estimator=self.estimator, **hyperparams)
-        self.imputer_.fit(df)
-        self.n_iter_ = self.imputer_.n_iter_
+        self.n_iter_ = 1
+        # requires fitting IterativeImputer in the fit method
+        # self.n_iter_ = self.imputer_.n_iter_
         return self
 
     def _transform_element(self, df: pd.DataFrame, col: str = "__all__") -> pd.DataFrame:
@@ -1358,12 +1356,10 @@ def _transform_element(self, df: pd.DataFrame, col: str = "__all__") -> pd.DataF
             Input has to be a pandas.DataFrame.
         """
         self._check_dataframe(df)
-        res = self.imputer_.transform(df)
-        imputed = pd.DataFrame(columns=df.columns)
-        for ind, col in enumerate(imputed.columns):
-            imputed[col] = res[:, ind]
-        imputed.index = df.index
-        return imputed
+        X_imputed = self.imputer_.fit_transform(df)
+        df_imputed = pd.DataFrame(X_imputed, index=df.index, columns=df.columns)
+
+        return df_imputed
 
 
 class ImputerRegressor(_Imputer):
@@ -1727,47 +1723,47 @@ def get_model(self, **hyperparams) -> em_sampler.EM:
                 " Valid values are `multinormal`and `VAR`."
             )
 
-    def fit(self, X: pd.DataFrame, y=None):
-        """Fit the imputer on X.
-
-        Parameters
-        ----------
-        X : pd.DataFrame
-            Data matrix on which the Imputer must be fitted.
-
-        Returns
-        -------
-        self : Self
-            Returns self.
-        """
-        super().fit(X)
-        df = self._check_input(X)
-
-        # n_rows, n_cols = df.shape
-        # if n_rows == 1:
-        #     raise ValueError("n_samples=1 is not allowed!")
-
-        if self.model not in ["multinormal", "VAR1"]:
-            raise ValueError(
-                f"Model argument `{self.model}` is invalid!"
-                " Valid values are `multinormal`and `VAR`."
-            )
-
-        cols_with_nans = df.columns[df.isna().any()]
-
-        self._models = {}
-        if self.columnwise:
-            for col in cols_with_nans:
-                hyperparams = self.get_hyperparams(col=col)
-                model = self.get_model(**hyperparams)
-                model.fit(df[col].values)
-                self._models[col] = model
-        else:
-            hyperparams = self.get_hyperparams()
-            model = self.get_model(**hyperparams)
-            model.fit(df.values.T)
-            self._models["__all__"] = model
-        return self
+    # def fit(self, X: pd.DataFrame, y=None):
+    #     """Fit the imputer on X.
+
+    #     Parameters
+    #     ----------
+    #     X : pd.DataFrame
+    #         Data matrix on which the Imputer must be fitted.
+
+    #     Returns
+    #     -------
+    #     self : Self
+    #         Returns self.
+    #     """
+    #     super().fit(X)
+    #     df = self._check_input(X)
+
+    #     # n_rows, n_cols = df.shape
+    #     # if n_rows == 1:
+    #     #     raise ValueError("n_samples=1 is not allowed!")
+
+    #     if self.model not in ["multinormal", "VAR1"]:
+    #         raise ValueError(
+    #             f"Model argument `{self.model}` is invalid!"
+    #             " Valid values are `multinormal`and `VAR`."
+    #         )
+
+    #     cols_with_nans = df.columns[df.isna().any()]
+
+    #     self._models = {}
+    #     if self.columnwise:
+    #         for col in cols_with_nans:
+    #             hyperparams = self.get_hyperparams(col=col)
+    #             model = self.get_model(**hyperparams)
+    #             model.fit(df[col].values)
+    #             self._models[col] = model
+    #     else:
+    #         hyperparams = self.get_hyperparams()
+    #         model = self.get_model(**hyperparams)
+    #         model.fit(df.values.T)
+    #         self._models["__all__"] = model
+    #     return self
 
     def _transform_element(self, df: pd.DataFrame, col: str = "__all__") -> pd.DataFrame:
         """
@@ -1792,7 +1788,13 @@ def _transform_element(self, df: pd.DataFrame, col: str = "__all__") -> pd.DataF
             Input has to be a pandas.DataFrame.
         """
         self._check_dataframe(df)
-        model = self._models[col]
+
+        hyperparams = self.get_hyperparams(col=col)
+        model = self.get_model(**hyperparams)
+        if col == "__all__":
+            model.fit(df.values.T)
+        else:
+            model.fit(df[col].values)
 
         X = df.values.T.astype(float)
         X_imputed = model.transform(X)

tests/imputations/test_imputers.py

@@ -3,9 +3,11 @@
 import numpy as np
 import pandas as pd
 import pytest
-from sklearn.ensemble import ExtraTreesRegressor
 from sklearn.utils.estimator_checks import check_estimator, parametrize_with_checks
 from qolmat.benchmark.hyperparameters import HyperValue
+from sklearn.linear_model import LinearRegression
+from sklearn.ensemble import ExtraTreesRegressor
+
 
 from qolmat.imputations import imputers
 
@@ -286,6 +288,35 @@ def test_ImputerEM_fit_transform(df: pd.DataFrame) -> None:
     np.testing.assert_allclose(result, expected, atol=1e-2)
 
 
+index_grouped = pd.MultiIndex.from_product([["a", "b"], range(4)], names=["group", "date"])
+dict_values = {"col1": [0, np.nan, 0, 0, 1, 1, 1, 1], "col2": np.arange(8)}
+df_grouped = pd.DataFrame(dict_values, index=index_grouped)
+
+list_imputers = [
+    imputers.ImputerMean(groups=("group",)),
+    imputers.ImputerMedian(groups=("group",)),
+    imputers.ImputerMode(groups=("group",)),
+    imputers.ImputerShuffle(groups=("group",)),
+    imputers.ImputerLOCF(groups=("group",)),
+    imputers.ImputerNOCB(groups=("group",)),
+    imputers.ImputerInterpolation(groups=("group",)),
+    imputers.ImputerResiduals(groups=("group",), period=2),
+    imputers.ImputerKNN(groups=("group",)),
+    imputers.ImputerMICE(groups=("group",)),
+    imputers.ImputerRegressor(groups=("group",), estimator=LinearRegression()),
+    imputers.ImputerRPCA(groups=("group",)),
+    imputers.ImputerEM(groups=("group",)),
+]
+
+
+@pytest.mark.parametrize("imputer", list_imputers)
+def test_models_fit_transform_grouped(imputer):
+    # imputer = imputers.ImputerEM(groups=("group",), method="sample", random_state=42)
+    result = imputer.fit_transform(df_grouped)
+    expected = df_grouped.fillna(0)
+    np.testing.assert_allclose(result, expected)
+
+
 @parametrize_with_checks(
     [
         imputers._Imputer(),
@@ -298,7 +329,7 @@ def test_ImputerEM_fit_transform(df: pd.DataFrame) -> None:
         imputers.ImputerNOCB(),
         imputers.ImputerInterpolation(),
         imputers.ImputerResiduals(period=2),
-        imputers.KNNImputer(),
+        imputers.ImputerKNN(),
         imputers.ImputerMICE(),
         imputers.ImputerRegressor(),
         imputers.ImputerRPCA(tau=0, lam=0),