merged

Author: Julien Roussel

HISTORY.rst

@@ -9,6 +9,7 @@ History
 * The Imputer classes do not possess a dictionary attribute anymore, and all list attributes have
 been changed into tuple attributes so that all are not immutable
 * All the tests from scikit-learn's check_estimator now pass for the class Imputer
+* Fix MLP imputer
 
 0.0.14 (2023-06-14)
 -------------------

examples/benchmark.md

@@ -76,26 +76,12 @@ The dataset `Beijing` is the Beijing Multi-Site Air-Quality Data Set. It consist
 This dataset only contains numerical vairables.
 
 ```python
-df_data = data.get_data_corrupted("Beijing", ratio_masked=.2, mean_size=20)
-
-# cols_to_impute = ["TEMP", "PRES", "DEWP", "NO2", "CO", "O3", "WSPM"]
-# cols_to_impute = df_data.columns[df_data.isna().any()]
+df_data = data.get_data_corrupted("Beijing", ratio_masked=.2, mean_size=120)
 cols_to_impute = ["TEMP", "PRES"]
-
 ```
 
 The dataset `Artificial` is designed to have a sum of a periodical signal, a white noise and some outliers.
 
-```python
-# df_data = data.get_data_corrupted("Artificial", ratio_masked=.2, mean_size=10)
-# cols_to_impute = ["signal"]
-```
-
-```python
-# df_data = data.get_data("SNCF", n_groups_max=2)
-# cols_to_impute = ["val_in"]
-```
-
 ```python
 df_data
 ```
@@ -244,7 +230,7 @@ comparison = comparator.Comparator(
     dict_imputers,
     cols_to_impute,
     generator_holes = generator_holes,
-    metrics=["mae", "wmape", "KL_columnwise", "ks_test", "energy"],
+    metrics=["mae", "wmape", "KL_columnwise", "ks_test"],
     max_evals=10,
     dict_config_opti=dict_config_opti,
 )
@@ -253,8 +239,14 @@ results
 ```
 
 ```python
-df_plot = results.loc["energy", "All"]
+df_plot = results.loc["KL_columnwise",'TEMP']
 plt.barh(df_plot.index, df_plot, color=tab10(0))
+plt.title('TEMP')
+plt.show()
+
+df_plot = results.loc["KL_columnwise",'PRES']
+plt.barh(df_plot.index, df_plot, color=tab10(0))
+plt.title('PRES')
 plt.show()
 ```
 
@@ -314,15 +306,19 @@ for col in cols_to_impute:
 
 ```
 
+```python
+dfs_imputed
+```
+
 ```python
 # plot.plot_imputations(df_station, dfs_imputed_station)
 
-n_columns = len(df_plot.columns)
+n_columns = len(cols_to_impute)
 n_imputers = len(dict_imputers)
 
 fig = plt.figure(figsize=(12 * n_imputers, 4 * n_columns))
 i_plot = 1
-for i_col, col in enumerate(df_plot):
+for i_col, col in enumerate(cols_to_impute):
     for name_imputer, df_imp in dfs_imputed_station.items():
 
         fig.add_subplot(n_columns, n_imputers, i_plot)
@@ -367,7 +363,6 @@ df = data.get_data("Beijing")
 cols_to_impute = ["TEMP", "PRES"]
 cols_with_nans = list(df.columns[df.isna().any()])
 df_data = data.add_datetime_features(df)
-df_data = data.add_station_features(df_data)
 df_data[cols_with_nans + cols_to_impute] = data.add_holes(pd.DataFrame(df_data[cols_with_nans + cols_to_impute]), ratio_masked=.1, mean_size=120)
 df_data
 ```
@@ -377,46 +372,38 @@ Then we train the model without taking a group on the stations
 
 ```python
 estimator = tf.keras.models.Sequential([
-    tf.keras.layers.Dense(256, activation='sigmoid'),
-    tf.keras.layers.Dense(128, activation='sigmoid'),
-    tf.keras.layers.Dense(64, activation='sigmoid'),
+    tf.keras.layers.Dense(256, activation='relu'),
+    tf.keras.layers.Dense(128, activation='relu'),
+    tf.keras.layers.Dense(64, activation='relu'),
     tf.keras.layers.Dense(1)])
-estimator.compile(optimizer='adam', loss='mse')
-dict_imputers["MLP"] = imputer_mlp = imputers_keras.ImputerRegressorKeras(estimator=estimator, handler_nan = "column")
+estimator.compile(optimizer='adam', loss='mae')
+dict_imputers["MLP"] = imputer_mlp = imputers_keras.ImputerRegressorKeras(estimator=estimator, groups=['station'], handler_nan = "column")
 ```
 
 We can re-run the imputation model benchmark as before.
-
-```python
-generator_holes = missing_patterns.EmpiricalHoleGenerator(n_splits=2, subset = cols_to_impute, ratio_masked=ratio_masked)
+```python jupyter={"outputs_hidden": true} tags=[]
+generator_holes = missing_patterns.EmpiricalHoleGenerator(n_splits=2, groups=["station"], subset=cols_to_impute, ratio_masked=ratio_masked)
 
 comparison = comparator.Comparator(
     dict_imputers,
-    df_data.columns,
+    cols_to_impute,
     generator_holes = generator_holes,
-    n_calls_opt=10,
+    metrics=["mae", "wmape", "KL_columnwise", "ks_test"],
+    max_evals=10,
     dict_config_opti=dict_config_opti,
 )
 results = comparison.compare(df_data)
 results
 ```
-
-```python
-fig = plt.figure(figsize=(24, 4))
-plot.multibar(results.loc["mae"], decimals=1)
-plt.ylabel("mae")
-plt.show()
-```
-
-```python
+```python jupyter={"outputs_hidden": true, "source_hidden": true} tags=[]
 df_plot = df_data
 dfs_imputed = {name: imp.fit_transform(df_plot) for name, imp in dict_imputers.items()}
 station = df_plot.index.get_level_values("station")[0]
 df_station = df_plot.loc[station]
 dfs_imputed_station = {name: df_plot.loc[station] for name, df_plot in dfs_imputed.items()}
 ```
 
-```python
+```python jupyter={"source_hidden": true} tags=[]
 for col in cols_to_impute:
     fig, ax = plt.subplots(figsize=(10, 3))
     values_orig = df_station[col]

qolmat/imputations/em_sampler.py

@@ -179,10 +179,6 @@ def fit(self, X: NDArray):
 
         # first imputation
         X_sample_last = utils.linear_interpolation(X)
-        print("X_sample_last")
-        print(X_sample_last)
-        print("x")
-        print(X)
         self.fit_distribution(X_sample_last)
 
         for iter_em in range(self.max_iter_em):
@@ -309,8 +305,6 @@ def fit_distribution(self, X):
             self.cov = np.eye(n_rows)
         else:
             self.cov = np.cov(X).reshape(n_rows, -1)
-        print("cov")
-        print(self.cov)
         self.cov_inv = np.linalg.pinv(self.cov, rcond=1e-2)
 
     def get_loglikelihood(self, X: NDArray) -> float:

qolmat/imputations/imputers.py

@@ -206,6 +206,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame:
         if df_imputed.isna().any().any():
             raise AssertionError("Result of imputation contains NaN!")
 
+        df_imputed = df_imputed.astype(float)
         if isinstance(X, (np.ndarray)):
             df_imputed = df_imputed.to_numpy()
 
@@ -1528,16 +1529,22 @@ def _transform_element(self, df: pd.DataFrame, col: str = "__all__") -> pd.DataF
             if col not in self.estimators_:
                 y_imputed = pd.Series(y.mean(), index=y.index)
             else:
-                y_imputed = self.estimators_[col].predict(X[is_na & is_valid])
-                y_imputed = pd.Series(y_imputed.flatten())
+                X_select = X[is_na & is_valid]
+                y_imputed = self.estimators_[col].predict(X_select)
+                y_imputed = y_imputed.flatten().astype(float)
+                print("y_imputed")
+                print(y_imputed)
+
+                y_imputed = pd.Series(y_imputed, index=X_select.index)
 
             # Adds the imputed values
-            df_imputed.loc[~is_na, col] = y[~is_na]
+            # df_imputed.loc[~is_na, col] = y[~is_na]
             # if isinstance(y_imputed, pd.Series):
             #     y_reshaped = y_imputed
             # else:
             #     y_reshaped = y_imputed.flatten()
-            df_imputed.loc[is_na & is_valid, col] = y_imputed.values[: sum(is_na & is_valid)]
+            # df_imputed.loc[is_na & is_valid, col] = y_imputed.values[: sum(is_na & is_valid)]
+            df_imputed[col] = y_imputed.where(is_valid & is_na, y)
 
         return df_imputed
 

tests/benchmark/test_hyperparameters.py

@@ -86,4 +86,4 @@ def test_hyperparameters_optimize():
         imputer, df, generator, metric, dict_config_opti, max_evals=500
     )
     assert isinstance(imputer_opti, ImputerTest)
-    np.testing.assert_almost_equal(imputer_opti.value, 4, decimal=2)
+    np.testing.assert_almost_equal(imputer_opti.value, 4, decimal=1)

tests/benchmark/test_missing_patterns.py

@@ -24,7 +24,7 @@
     "unif": mp.UniformHoleGenerator(n_splits=2, ratio_masked=0.1, random_state=42),
     "multi": mp.MultiMarkovHoleGenerator(n_splits=2, ratio_masked=0.1, random_state=42),
     "group": mp.GroupedHoleGenerator(
-        n_splits=2, ratio_masked=0.1, random_state=42, groups=["group"]
+        n_splits=2, ratio_masked=0.1, random_state=42, groups=("group",)
     ),
 }
 

tests/imputations/test_imputers_keras.py

@@ -43,14 +43,22 @@ def test_ImputerRegressorKeras_fit_transform(df: pd.DataFrame) -> None:
     )
 
     result = imputer.fit_transform(df)
+    # expected = pd.DataFrame(
+    #     {
+    #         "col1": [0.002, 15.0, 0.002, 23.0, 33.0],
+    #         "col2": [69.0, 76.0, 74.0, 80.0, 78.0],
+    #         "col3": [174.0, 166.0, 182.0, 177.0, 0.003714],
+    #         "col4": [9.0, 12.0, 11.0, 12.0, 8.0],
+    #         "col5": [93.0, 75.0, 0.005459, 12.0, 0.005461],
+    #     }
+    # )
     expected = pd.DataFrame(
         {
-            "col1": [38.362286, 15.0, 38.365032, 23.0, 33.0],
+            "col1": [38.363, 15.0, 38.365, 23.0, 33.0],
             "col2": [69.0, 76.0, 74.0, 80.0, 78.0],
-            "col3": [174.0, 166.0, 182.0, 177.0, 38.365231],
+            "col3": [174.0, 166.0, 182.0, 177.0, 38.365],
             "col4": [9.0, 12.0, 11.0, 12.0, 8.0],
-            "col5": [93.0, 75.0, 38.365032, 12.0, 38.365269],
+            "col5": [93.0, 75.0, 38.365, 12.0, 38.365],
         }
     )
-
-    np.testing.assert_allclose(result["col3"], expected["col3"], atol=1e-3)
+    pd.testing.assert_frame_equal(result, expected, atol=1e-3)