code review: parameter method added

Author: vm-aifluence-jro

.pre-commit-config.yaml

@@ -23,4 +23,3 @@ repos:
     rev: v1.1.1
     hooks:
       - id: mypy
-        exclude: (tests/old_tests)

examples/benchmark.md

@@ -144,7 +144,7 @@ imputer_mice = imputers.ImputerMICE(groups=["station"], estimator=LinearRegressi
 imputer_regressor = imputers.ImputerRegressor(groups=["station"], estimator=LinearRegression())
 
 dict_imputers = {
-    # "mean": imputer_mean,
+    "mean": imputer_mean,
     # "median": imputer_median,
     # "mode": imputer_mode,
     "interpolation": imputer_interpol,
@@ -160,7 +160,7 @@ dict_imputers = {
     # "nocb": imputer_nocb,
     # "knn": imputer_knn,
     "ols": imputer_regressor,
-    "mice_ols": imputer_mice,
+    # "mice_ols": imputer_mice,
 }
 n_imputers = len(dict_imputers)
 
@@ -193,14 +193,24 @@ comparison = comparator.Comparator(
     dict_imputers,
     cols_to_impute,
     generator_holes = generator_holes,
-    metrics=["mae", "wmape", "KL"],
+    metrics=["mae", "wmape", "KL", "ks_test", "energy"],
     n_calls_opt=10,
     search_params=search_params,
 )
 results = comparison.compare(df_data)
 results
 ```
 
+```python
+df_plot
+```
+
+```python
+df_plot = results.loc["energy", "All"]
+plt.bar(df_plot.index, df_plot, color=tab10(0))
+plt.show()
+```
+
 ```python
 fig = plt.figure(figsize=(24, 8))
 fig.add_subplot(2, 1, 1)

examples/metrics_usage.md

@@ -28,7 +28,7 @@ from sklearn.linear_model import LinearRegression
 from qolmat.utils import data, plot, utils
 from qolmat.imputations import imputers
 from qolmat.benchmark import comparator, missing_patterns
-from qolmat.benchmark.utils import wasser_distance, kl_divergence, frechet_distance
+from qolmat.benchmark.utils import wasser_distance_columnwise, kl_divergence, frechet_distance
 ```
 
 ```python
@@ -106,7 +106,7 @@ ratio_masked = 0.1
 ```python
 # Métriques
 metrics = {
-    "wasser": wasser_distance,
+    "wasserstein_columnwise": wasserstein_distance_columnwise,
     "KL": kl_divergence
     #"frechet": frechet_distance
 }

qolmat/benchmark/comparator.py

@@ -1,4 +1,5 @@
 import logging
+from functools import partial
 from typing import Any, Dict, List, Optional, Union
 
 import numpy as np
@@ -33,8 +34,9 @@ class Comparator:
         "rmse": metrics.root_mean_squared_error,
         "mae": metrics.mean_absolute_error,
         "wmape": metrics.weighted_mean_absolute_percentage_error,
-        "wasser": metrics.wasser_distance,
-        "KL": metrics.kl_divergence_columnwise,
+        "wasserstein_columnwise": partial(metrics.wasserstein_distance, method="columnwise"),
+        "KL_columnwise": partial(metrics.kl_divergence, method="columnwise"),
+        "KL_gaussian": partial(metrics.kl_divergence, method="gaussian"),
         "ks_test": metrics.kolmogorov_smirnov_test,
         "correlation_diff": metrics.mean_difference_correlation_matrix_numerical_features,
         "pairwise_dist": metrics.sum_pairwise_distances,

qolmat/benchmark/metrics.py

@@ -3,7 +3,6 @@
 import numpy as np
 import pandas as pd
 import scipy
-
 from sklearn import metrics as skm
 from sklearn.preprocessing import StandardScaler
 
@@ -105,7 +104,9 @@ def weighted_mean_absolute_percentage_error(
     return columnwise_metric(df1, df2, df_mask, skm.mean_absolute_percentage_error)
 
 
-def wasser_distance(df1: pd.DataFrame, df2: pd.DataFrame, df_mask: pd.DataFrame) -> pd.Series:
+def wasserstein_distance(
+    df1: pd.DataFrame, df2: pd.DataFrame, df_mask: pd.DataFrame, method: str = "columnwise"
+) -> pd.Series:
     """Wasserstein distances between columns of 2 dataframes.
     Wasserstein distance can only be computed columnwise
 
@@ -122,7 +123,13 @@ def wasser_distance(df1: pd.DataFrame, df2: pd.DataFrame, df_mask: pd.DataFrame)
     -------
     wasserstein distances : pd.Series
     """
-    return columnwise_metric(df1, df2, df_mask, scipy.stats.wasserstein_distance)
+    if method == "columnwise":
+        return columnwise_metric(df1, df2, df_mask, scipy.stats.wasserstein_distance)
+    else:
+        raise AssertionError(
+            f"The parameter of the function wasserstein_distance should be one of"
+            f"the following: [`columnwise`], not `{method}`!"
+        )
 
 
 def kl_divergence_1D(df1: pd.Series, df2: pd.Series) -> np.number:
@@ -134,30 +141,9 @@ def kl_divergence_1D(df1: pd.Series, df2: pd.Series) -> np.number:
     return scipy.stats.entropy(p + EPS, q + EPS)
 
 
-def kl_divergence_columnwise(
-    df1: pd.DataFrame, df2: pd.DataFrame, df_mask: pd.DataFrame
+def kl_divergence(
+    df1: pd.DataFrame, df2: pd.DataFrame, df_mask: pd.DataFrame, method: str = "columnwise"
 ) -> pd.Series:
-    """TODO documentation
-    Kullback-Leibler divergence between distributions
-    If multivariate normal distributions:
-    https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence
-
-    Parameters
-    ----------
-    df1 : pd.DataFrame
-    df2 : pd.DataFrame
-    columnwise_evaluation: Optional[bool]
-        if the evalutation is computed column-wise. By default, is set to False
-
-    Returns
-    -------
-    Kullback-Leibler divergence : Union[float, pd.Series]
-    """
-
-    return columnwise_metric(df1, df2, df_mask, kl_divergence_1D)
-
-
-def kl_divergence(df1: pd.DataFrame, df2: pd.DataFrame, df_mask: pd.DataFrame) -> pd.Series:
     """TODO Documentation
     Kullback-Leibler divergence between distributions
     If multivariate normal distributions:
@@ -174,22 +160,30 @@ def kl_divergence(df1: pd.DataFrame, df2: pd.DataFrame, df_mask: pd.DataFrame) -
     -------
     Kullback-Leibler divergence : Union[float, pd.Series]
     """
-    cols = df1.columns.tolist()
-    df_1 = StandardScaler().fit_transform(df1[df_mask.any(axis=1)])
-    df_2 = StandardScaler().fit_transform(df2[df_mask.any(axis=1)])
-
-    n = df_1.shape[0]
-    mu_true = np.nanmean(df_1, axis=0)
-    sigma_true = np.ma.cov(np.ma.masked_invalid(df_1), rowvar=False).data
-    mu_pred = np.nanmean(df_2, axis=0)
-    sigma_pred = np.ma.cov(np.ma.masked_invalid(df_2), rowvar=False).data
-    diff = mu_true - mu_pred
-    inv_sigma_pred = np.linalg.inv(sigma_pred)
-    quad_term = diff.T @ inv_sigma_pred @ diff
-    trace_term = np.trace(inv_sigma_pred @ sigma_true)
-    det_term = np.log(np.linalg.det(sigma_pred) / np.linalg.det(sigma_true))
-    kl = 0.5 * (quad_term + trace_term + det_term - n)
-    return pd.Series(kl, index=cols)
+    if method == "columnwise":
+        return columnwise_metric(df1, df2, df_mask, kl_divergence_1D)
+    elif method == "gaussian":
+        cols = df1.columns.tolist()
+        df_1 = StandardScaler().fit_transform(df1[df_mask.any(axis=1)])
+        df_2 = StandardScaler().fit_transform(df2[df_mask.any(axis=1)])
+
+        n = df_1.shape[0]
+        mu_true = np.nanmean(df_1, axis=0)
+        sigma_true = np.ma.cov(np.ma.masked_invalid(df_1), rowvar=False).data
+        mu_pred = np.nanmean(df_2, axis=0)
+        sigma_pred = np.ma.cov(np.ma.masked_invalid(df_2), rowvar=False).data
+        diff = mu_true - mu_pred
+        inv_sigma_pred = np.linalg.inv(sigma_pred)
+        quad_term = diff.T @ inv_sigma_pred @ diff
+        trace_term = np.trace(inv_sigma_pred @ sigma_true)
+        det_term = np.log(np.linalg.det(sigma_pred) / np.linalg.det(sigma_true))
+        kl = 0.5 * (quad_term + trace_term + det_term - n)
+        return pd.Series(kl, index=cols)
+    else:
+        raise AssertionError(
+            f"The parameter of the function wasserstein_distance should be one of"
+            f"the following: [`columnwise`, `gaussian`], not `{method}`!"
+        )
 
 
 def _get_numerical_features(df1: pd.DataFrame) -> List[str]:
@@ -242,7 +236,7 @@ def _get_categorical_features(df1: pd.DataFrame) -> List[str]:
         return cols_categorical
 
 
-def _kolmogorov_smirnov_test(df1: pd.Series, df2: pd.Series) -> float:
+def kolmogorov_smirnov_test_1D(df1: pd.Series, df2: pd.Series) -> float:
     """Compute KS test statistic of the two-sample Kolmogorov-Smirnov test for goodness of fit.
     See more in https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.ks_2samp.html.
 
@@ -283,11 +277,14 @@ def kolmogorov_smirnov_test(
     """
     cols_numerical = _get_numerical_features(df1)
     return columnwise_metric(
-        df1[cols_numerical], df2[cols_numerical], df_mask[cols_numerical], _kolmogorov_smirnov_test
+        df1[cols_numerical],
+        df2[cols_numerical],
+        df_mask[cols_numerical],
+        kolmogorov_smirnov_test_1D,
     )
 
 
-def _total_variance_distance(df1: pd.Series, df2: pd.Series) -> float:
+def total_variance_distance_1D(df1: pd.Series, df2: pd.Series) -> float:
     """Compute Total Variance Distance for a categorical feature
     It is based on TVComplement in https://github.com/sdv-dev/SDMetrics
 
@@ -337,7 +334,7 @@ def total_variance_distance(
         df1[cols_categorical],
         df2[cols_categorical],
         df_mask[cols_categorical],
-        _total_variance_distance,
+        total_variance_distance_1D,
     )
 
 
@@ -564,20 +561,20 @@ def mean_difference_correlation_matrix_categorical_vs_numerical_features(
 
 
 ###########################
-# Row-wise metris         #
+# Row-wise metrics        #
 ###########################
 
 
-def _sum_distance_col(col: pd.Series, col_size: int) -> float:
-    col = col.sort_values(ascending=True)
-    sums_partial = col.shift().fillna(0.0).cumsum()
-    differences_partial = col * np.arange(col_size) - sums_partial
+def _sum_manhattan_distances_1D(values: pd.Series) -> float:
+    values = values.sort_values(ascending=True)
+    sums_partial = values.shift().fillna(0.0).cumsum()
+    differences_partial = values * np.arange(len(values)) - sums_partial
     res = differences_partial.sum()
     return res
 
 
 def _sum_manhattan_distances(df1: pd.DataFrame) -> float:
-    """Sum Manhattan distances.
+    """Sum Manhattan distances beetween all pairs of rows.
     It is based on https://www.geeksforgeeks.org/sum-manhattan-distances-pairs-points/
 
     Parameters
@@ -586,10 +583,8 @@ def _sum_manhattan_distances(df1: pd.DataFrame) -> float:
         _description_
     """
     cols = df1.columns.tolist()
-    sum = 0.0
-    for col in cols:
-        sum += _sum_distance_col(df1[col], len(df1))
-    return sum
+    result = sum([_sum_manhattan_distances_1D(df1[col]) for col in cols])
+    return result
 
 
 def sum_energy_distances(df1: pd.DataFrame, df2: pd.DataFrame, df_mask: pd.DataFrame) -> pd.Series:
@@ -613,9 +608,8 @@ def sum_energy_distances(df1: pd.DataFrame, df2: pd.DataFrame, df_mask: pd.DataF
     df1 = df1[df_mask].fillna(0.0)
     df2 = df2[df_mask].fillna(0.0)
 
-    sum_distances_df1 = _sum_manhattan_distances(
-        df1
-    )  # sum of (len_df1 * (len_df1 - 1) / 2) distances for df1
+    # sum of (len_df1 * (len_df1 - 1) / 2) distances for df1
+    sum_distances_df1 = _sum_manhattan_distances(df1)
     sum_distances_df2 = _sum_manhattan_distances(df2)
 
     df = pd.concat([df1, df2])
@@ -654,7 +648,7 @@ def sum_pairwise_distances(
 
 
 ###########################
-# Dataframe-wise metris   #
+# Dataframe-wise metrics  #
 ###########################
 
 

tests/benchmark/test_metrics.py

@@ -2,10 +2,10 @@
 # # Evaluation metrics #
 # ######################
 
-import pandas as pd
 import numpy as np
-import scipy
+import pandas as pd
 import pytest
+import scipy
 
 from qolmat.benchmark import metrics
 
@@ -71,25 +71,19 @@ def test_weighted_mean_absolute_percentage_error(
     assert metrics.weighted_mean_absolute_percentage_error(df1, df1, df_mask).equals(
         pd.Series([0.0, 0.0], index=["col1", "col2"])
     )
-    assert (
-        metrics.weighted_mean_absolute_percentage_error(df1, df2, df_mask)
-        .round(3)
-        .equals(pd.Series([0.083, 1.167], index=["col1", "col2"]))
-    )
+    result = metrics.weighted_mean_absolute_percentage_error(df1, df2, df_mask)
+    expected = pd.Series([0.083, 1.167], index=["col1", "col2"])
+    np.testing.assert_allclose(result, expected, atol=1e-3)
 
 
 @pytest.mark.parametrize("df1", [df_incomplete])
 @pytest.mark.parametrize("df2", [df_imputed])
 @pytest.mark.parametrize("df_mask", [df_mask])
-def test_wasser_distance(df1: pd.DataFrame, df2: pd.DataFrame, df_mask: pd.DataFrame) -> None:
-    assert metrics.wasser_distance(df1, df1, df_mask).equals(
-        pd.Series([0.0, 0.0], index=["col1", "col2"])
-    )
-    assert (
-        metrics.wasser_distance(df1, df2, df_mask)
-        .round(3)
-        .equals(pd.Series([0.250, 0.833], index=["col1", "col2"]))
-    )
+def test_wasserstein_distance(df1: pd.DataFrame, df2: pd.DataFrame, df_mask: pd.DataFrame) -> None:
+    dist = metrics.wasserstein_distance(df1, df1, df_mask, method="columnwise")
+    assert dist.equals(pd.Series([0.0, 0.0], index=["col1", "col2"]))
+    dist = metrics.wasserstein_distance(df1, df2, df_mask, method="columnwise")
+    assert dist.round(3).equals(pd.Series([0.250, 0.833], index=["col1", "col2"]))
 
 
 @pytest.mark.parametrize("df1", [df_incomplete])
@@ -98,27 +92,25 @@ def test_wasser_distance(df1: pd.DataFrame, df2: pd.DataFrame, df_mask: pd.DataF
 def test_kl_divergence_columnwise(
     df1: pd.DataFrame, df2: pd.DataFrame, df_mask: pd.DataFrame
 ) -> None:
-    assert metrics.kl_divergence_columnwise(df1, df1, df_mask).equals(
+    assert metrics.kl_divergence(df1, df1, df_mask, method="columnwise").equals(
         pd.Series([0.0, 0.0], index=["col1", "col2"])
     )
-    assert (
-        metrics.kl_divergence_columnwise(df1, df2, df_mask)
-        .round(3)
-        .equals(pd.Series([18.945, 36.637], index=["col1", "col2"]))
-    )
+    result = metrics.kl_divergence(df1, df2, df_mask, method="columnwise")
+    expected = pd.Series([18.945, 36.637], index=["col1", "col2"])
+    np.testing.assert_allclose(result, expected, atol=1e-3)
 
 
 @pytest.mark.parametrize("df1", [df_incomplete])
 @pytest.mark.parametrize("df2", [df_imputed])
 @pytest.mark.parametrize("df_mask", [df_mask])
 def test_kl_divergence(df1: pd.DataFrame, df2: pd.DataFrame, df_mask: pd.DataFrame) -> None:
     assert (
-        metrics.kl_divergence(df1, df1, df_mask)
+        metrics.kl_divergence(df1, df1, df_mask, method="gaussian")
         .round(2)
         .equals(pd.Series([-0.5, -0.5], index=["col1", "col2"]))
     )
     assert (
-        metrics.kl_divergence(df1, df2, df_mask)
+        metrics.kl_divergence(df1, df2, df_mask, method="gaussian")
         .round(3)
         .equals(pd.Series([0.263, 0.263], index=["col1", "col2"]))
     )

tests/imputations/test_imputers.py

@@ -154,7 +154,7 @@ def test_ImputerResiduals_fit_transform(df: pd.DataFrame) -> None:
         },
         index=pd.date_range("2023-04-17", periods=20, freq="D"),
     )
-    np.testing.assert_allclose(result, expected, rtol=1e-6)
+    np.testing.assert_allclose(result, expected, atol=1e-6)
 
 
 @pytest.mark.parametrize("df", [df_incomplete])
@@ -228,4 +228,4 @@ def test_ImputerEM_fit_transform(df: pd.DataFrame) -> None:
             "col2": [0, 1.914706, 2, 2.480963, 2] + [i for i in range(5, 20)],
         }
     )
-    np.testing.assert_allclose(result, expected, rtol=1e-6)
+    np.testing.assert_allclose(result, expected, atol=1e-6)