init completed

Author: Julien Roussel

examples/benchmark.md

@@ -64,11 +64,10 @@ This dataset only contains numerical vairables.
 
 ```python
 df_data = data.get_data_corrupted("Beijing", ratio_masked=.2, mean_size=120)
-df_data["cat"] = [i % 3 for i in range(len(df_data))]
 
 # cols_to_impute = ["TEMP", "PRES", "DEWP", "NO2", "CO", "O3", "WSPM"]
 # cols_to_impute = df_data.columns[df_data.isna().any()]
-cols_to_impute = ["TEMP", "PRES", "cat"]
+cols_to_impute = ["TEMP", "PRES"]
 
 ```
 
@@ -113,13 +112,9 @@ All presented methods are group-wise: here each station is imputed independently
 Some methods require hyperparameters. The user can directly specify them, or rather determine them through an optimization step using the `search_params` dictionary. The keys are the imputation method's name and the values are a dictionary specifying the minimum, maximum or list of categories and type of values (Integer, Real, Category or a dictionary indexed by the variable names) to search.
 In pratice, we rely on a cross validation to find the best hyperparams values minimizing an error reconstruction.
 
-```python tags=[]
-hasattr(imputers.ImputerMean(), "groups")
-```
-
 ```python
 imputer_mean = imputers.ImputerMean(groups=["station"])
-imputer_median = imputers.ImputerMedian(groups=["station", "cat"])
+imputer_median = imputers.ImputerMedian(groups=["station"])
 imputer_mode = imputers.ImputerMode(groups=["station"])
 imputer_locf = imputers.ImputerLOCF(groups=["station"])
 imputer_nocb = imputers.ImputerNOCB(groups=["station"])
@@ -248,6 +243,8 @@ for col in cols_to_impute:
 ```
 
 ```python
+# plot.plot_imputations(df_station, dfs_imputed_station)
+
 n_columns = len(df_plot.columns)
 n_imputers = len(dict_imputers)
 
@@ -272,7 +269,6 @@ for name_imputer in dict_imputers:
         ax.xaxis.set_major_locator(loc)
         ax.tick_params(axis='both', which='major', labelsize=17)
         i_plot += 1
-        plt.xlim(0, 100)
 plt.savefig("figures/imputations_benchmark.png")
 plt.show()
 

examples/figures/imputations_benchmark.png

@@ -1,5 +1,4 @@
+from . import benchmark, imputations, utils
 from ._version import __version__
 
-from . import utils
-
 __all__ = ["utils", "__version__"]

qolmat/__init__.py

@@ -105,10 +105,10 @@ def impute_element(self, df: pd.DataFrame) -> pd.DataFrame:
             imputation_values = self.fit_transform_element(df)
 
         df = df.fillna(imputation_values)
-        # # fill na by applying imputation method without groups
-        # if df.isna().any().any():
-        #     imputation_values = self.fit_transform_fallback(df)
-        #     df = df.fillna(imputation_values)
+        # fill na by applying imputation method without groups
+        if df.isna().any().any():
+            imputation_values = self.fit_transform_fallback(df)
+            df = df.fillna(imputation_values)
 
         return df
 

qolmat/imputations/imputers.py

@@ -4,10 +4,11 @@
 
 from __future__ import annotations
 
-from typing import List, Optional, Tuple, Union
+from typing import Dict, List, Optional, Tuple, Union
 
 import matplotlib as mpl
 import matplotlib.pyplot as plt
+import matplotlib.ticker as plticker
 import numpy as np
 import pandas as pd
 import scipy
@@ -256,3 +257,31 @@ def multibar(df, ax=None, orientation="vertical", colors=None, decimals=0):
     # ax.bar_label(rects2, padding=3)
 
     # plt.tight_layout()
+
+
+def plot_imputations(df: pd.DataFrame, dict_df_imputed: Dict[str, pd.DataFrame]):
+    n_columns = len(df.columns)
+    n_imputers = len(dict_df_imputed)
+
+    fig = plt.figure(figsize=(8 * n_columns, 6 * n_imputers))
+    i_plot = 1
+    for name_imputer, df_imputed in dict_df_imputed.items():
+        for col in df:
+
+            ax = fig.add_subplot(n_imputers, n_columns, i_plot)
+            values_orig = df[col]
+
+            plt.plot(values_orig, ".", color="black", label="original")
+            # plt.plot(df.iloc[870:1000][col], markers[0], color='k', linestyle='-' , ms=3)
+
+            values_imp = df_imputed[col].copy()
+            values_imp[values_orig.notna()] = np.nan
+            plt.plot(values_imp, ".", color=tab10(0), label=name_imputer, alpha=1)
+            plt.ylabel(col, fontsize=16)
+            if i_plot % n_columns == 0:
+                plt.legend(loc=[1, 0], fontsize=18)
+            loc = plticker.MultipleLocator(base=2 * 365)
+            ax.xaxis.set_major_locator(loc)
+            ax.tick_params(axis="both", which="major", labelsize=17)
+            i_plot += 1
+    plt.show()