comparator holes are now identical for all imputers

Author: Julien Roussel

qolmat/benchmark/comparator.py

@@ -5,6 +5,7 @@
 
 import numpy as np
 import pandas as pd
+from sklearn import utils as sku
 
 from qolmat.benchmark import hyperparameters, metrics
 from qolmat.benchmark.missing_patterns import _HoleGenerator
@@ -169,8 +170,12 @@ def compare(
 
         """
         dict_errors = {}
-
+        self.generator_holes.random_state = sku.check_random_state(
+            self.generator_holes.random_state
+        )
+        self.generator_holes.save_rng_state()
         for name, imputer in self.dict_imputers.items():
+            self.generator_holes.load_rng_state()
             dict_config_opti_imputer = self.dict_config_opti.get(name, {})
 
             try:

qolmat/benchmark/missing_patterns.py

@@ -190,6 +190,12 @@ def _check_subset(self, X: pd.DataFrame):
         elif isinstance(self.subset, str):
             raise SubsetIsAString(self.subset)
 
+    def save_rng_state(self):
+        self.state_rng = self.random_state.get_state()
+
+    def load_rng_state(self):
+        self.random_state.set_state(self.state_rng)
+
 
 class UniformHoleGenerator(_HoleGenerator):
     """UniformHoleGenerator class.

qolmat/imputations/em_sampler.py

@@ -190,7 +190,6 @@ def __init__(
         self.n_iter_ou = n_iter_ou
         self.ampli = ampli
         self.rng = sku.check_random_state(random_state)
-        self.cov = np.array([[]])
         self.dt = dt
         self.tolerance = tolerance
         self.stagnation_threshold = stagnation_threshold
@@ -657,6 +656,7 @@ def __init__(
             period=period,
             verbose=verbose,
         )
+        self.cov = np.array([[]])
         self.dict_criteria_stop = {"logliks": [], "means": [], "covs": []}
 
     def get_loglikelihood(self, X: NDArray) -> float:

qolmat/utils/input_check.py

@@ -1,4 +1,5 @@
 """Util file for input checks."""
+
 import pandas as pd
 
 from qolmat.utils.exceptions import TypeNotHandled

qolmat/utils/utils.py

@@ -33,6 +33,7 @@ def _get_numerical_features(df1: pd.DataFrame) -> List[str]:
     """
     cols_numerical = df1.select_dtypes(include=np.number).columns.tolist()
     if len(cols_numerical) == 0:
+        print(df1)
         raise Exception("No numerical feature is found.")
     else:
         return cols_numerical

tests/analysis/test_holes_characterization.py

@@ -2,6 +2,7 @@
 import pandas as pd
 import pytest
 from scipy.stats import norm
+from sklearn import utils as sku
 
 from qolmat.analysis.holes_characterization import LittleTest, PKLMTest
 from qolmat.benchmark.missing_patterns import UniformHoleGenerator
@@ -12,7 +13,7 @@
 
 @pytest.fixture
 def mcar_df() -> pd.DataFrame:
-    rng = np.random.default_rng(42)
+    rng = sku.check_random_state(42)
     matrix = rng.multivariate_normal(
         mean=[0, 0], cov=[[1, 0], [0, 1]], size=200
     )
@@ -26,7 +27,7 @@ def mcar_df() -> pd.DataFrame:
 
 @pytest.fixture
 def mar_hm_df() -> pd.DataFrame:
-    rng = np.random.default_rng(42)
+    rng = sku.check_random_state(42)
     matrix = rng.multivariate_normal(
         mean=[0, 0], cov=[[1, 0], [0, 1]], size=200
     )
@@ -42,7 +43,7 @@ def mar_hm_df() -> pd.DataFrame:
 
 @pytest.fixture
 def mar_hc_df() -> pd.DataFrame:
-    rng = np.random.default_rng(42)
+    rng = sku.check_random_state(42)
     matrix = rng.multivariate_normal(
         mean=[0, 0], cov=[[1, 0], [0, 1]], size=200
     )
@@ -88,7 +89,7 @@ def supported_multitypes_dataframe() -> pd.DataFrame:
 
 @pytest.fixture
 def np_matrix_with_nan_mcar() -> np.ndarray:
-    rng = np.random.default_rng(42)
+    rng = sku.check_random_state(42)
     n_rows, n_cols = 10, 4
     matrix = rng.normal(size=(n_rows, n_cols))
     num_nan = int(n_rows * n_cols * 0.40)
@@ -104,7 +105,7 @@ def missingness_matrix_mcar(np_matrix_with_nan_mcar):
 
 @pytest.fixture
 def missingness_matrix_mcar_perm(missingness_matrix_mcar):
-    rng = np.random.default_rng(42)
+    rng = sku.check_random_state(42)
     return rng.permutation(missingness_matrix_mcar)
 
 

tests/benchmark/test_comparator.py

@@ -2,20 +2,28 @@
 
 import numpy as np
 import pandas as pd
+import pytest
 
 from qolmat.benchmark.comparator import Comparator
+from qolmat.benchmark.missing_patterns import UniformHoleGenerator
+from qolmat.imputations.imputers import ImputerShuffle
 
-generator_holes_mock = MagicMock()
-generator_holes_mock.split.return_value = [
-    pd.DataFrame({"A": [False, False, True], "B": [True, False, False]})
-]
 
-comparator = Comparator(
-    dict_models={},
-    selected_columns=["A", "B"],
-    generator_holes=generator_holes_mock,
-    metrics=["mae", "mse"],
-)
+@pytest.fixture
+def comparator_fix():
+    generator_holes_mock = MagicMock()
+    generator_holes_mock.split.return_value = [
+        pd.DataFrame({"A": [False, False, True], "B": [True, False, False]})
+    ]
+    generator_holes_mock.random_state = 0
+    comparator = Comparator(
+        dict_models={},
+        selected_columns=["A", "B"],
+        generator_holes=generator_holes_mock,
+        metrics=["mae", "mse"],
+    )
+    return comparator
+
 
 imputer_mock = MagicMock()
 expected_get_errors = pd.Series(
@@ -27,7 +35,7 @@
 
 
 @patch("qolmat.benchmark.metrics.get_metric")
-def test_get_errors(mock_get_metric):
+def test_get_errors(mock_get_metric, comparator_fix):
     df_origin = pd.DataFrame({"A": [1, np.nan, 3], "B": [np.nan, 5, 6]})
     df_imputed = pd.DataFrame({"A": [1, 2, 4], "B": [4, 5, 7]})
     df_mask = pd.DataFrame(
@@ -39,7 +47,7 @@ def test_get_errors(mock_get_metric):
             [1.0, 1.0], index=["A", "B"]
         )
     )
-    errors = comparator.get_errors(df_origin, df_imputed, df_mask)
+    errors = comparator_fix.get_errors(df_origin, df_imputed, df_mask)
     pd.testing.assert_series_equal(errors, expected_get_errors)
 
 
@@ -48,8 +56,10 @@ def test_get_errors(mock_get_metric):
     "qolmat.benchmark.comparator.Comparator.get_errors",
     return_value=expected_get_errors,
 )
-def test_evaluate_errors_sample(mock_get_errors, mock_optimize):
-    errors_mean = comparator.evaluate_errors_sample(
+def test_evaluate_errors_sample(
+    mock_get_errors, mock_optimize, comparator_fix
+):
+    errors_mean = comparator_fix.evaluate_errors_sample(
         imputer_mock, pd.DataFrame({"A": [1, 2, 3], "B": [4, 5, np.nan]})
     )
     expected_errors_mean = expected_get_errors
@@ -62,12 +72,12 @@ def test_evaluate_errors_sample(mock_get_errors, mock_optimize):
     "qolmat.benchmark.comparator.Comparator.evaluate_errors_sample",
     return_value=expected_get_errors,
 )
-def test_compare(mock_evaluate_errors_sample):
+def test_compare(mock_evaluate_errors_sample, comparator_fix):
     df_test = pd.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})
 
     imputer1 = MagicMock(name="Imputer1")
     imputer2 = MagicMock(name="Imputer2")
-    comparator.dict_imputers = {"imputer1": imputer1, "imputer2": imputer2}
+    comparator_fix.dict_imputers = {"imputer1": imputer1, "imputer2": imputer2}
 
     errors_imputer1 = pd.Series([0.1, 0.2], index=["mae", "mse"])
     errors_imputer2 = pd.Series([0.3, 0.4], index=["mae", "mse"])
@@ -76,7 +86,7 @@ def test_compare(mock_evaluate_errors_sample):
         errors_imputer2,
     ]
 
-    df_errors = comparator.compare(df_test)
+    df_errors = comparator_fix.compare(df_test)
     assert mock_evaluate_errors_sample.call_count == 2
 
     mock_evaluate_errors_sample.assert_any_call(imputer1, df_test, {}, "mse")
@@ -85,3 +95,28 @@ def test_compare(mock_evaluate_errors_sample):
         {"imputer1": [0.1, 0.2], "imputer2": [0.3, 0.4]}, index=["mae", "mse"]
     )
     pd.testing.assert_frame_equal(df_errors, expected_df_errors)
+
+
+def test_compare_reproducibility():
+    seed = 123
+    dict_models = {
+        "shuffle1": ImputerShuffle(random_state=seed),
+        "shuffle2": ImputerShuffle(random_state=seed),
+    }
+    cols = ["A", "B"]
+    df_data = pd.DataFrame(
+        np.random.random((100, 2)), dtype=float, columns=cols
+    )
+    generator_holes = UniformHoleGenerator(
+        n_splits=2, subset=cols, ratio_masked=0.5
+    )
+    comparator = Comparator(
+        dict_models=dict_models,
+        selected_columns=df_data.columns,
+        generator_holes=generator_holes,
+        metrics=["mae", "mse"],
+    )
+    df_errors = comparator.compare(df_data)
+    pd.testing.assert_series_equal(
+        df_errors["shuffle1"], df_errors["shuffle2"], check_names=False
+    )

tests/imputations/test_em_sampler.py

@@ -5,6 +5,7 @@
 import scipy
 from numpy.typing import NDArray
 from scipy import linalg
+from sklearn import utils as sku
 from sklearn.datasets import make_spd_matrix
 
 from qolmat.imputations import em_sampler
@@ -31,8 +32,8 @@
 
 # @pytest.fixture
 def generate_multinormal_predefined_mean_cov(d=3, n=500):
-    rng = np.random.default_rng(42)
-    seed = rng.integers(np.iinfo(np.int32).max)
+    rng = sku.check_random_state(42)
+    seed = rng.randint(np.iinfo(np.int32).max)
     random_state = np.random.RandomState(seed=seed)
     mean = np.array([rng.uniform(low=0, high=d) for _ in range(d)])
     covariance = make_spd_matrix(n_dim=d, random_state=random_state)
@@ -51,7 +52,7 @@ def generate_multinormal_predefined_mean_cov(d=3, n=500):
 
 
 def get_matrix_B(d, p, eigmax=1):
-    rng = np.random.default_rng(42)
+    rng = sku.check_random_state(42)
     B = rng.normal(0, 1, size=(d * p + 1, d))
     U, S, Vt = linalg.svd(B, check_finite=False, full_matrices=False)
     S = rng.uniform(0, eigmax, size=d)
@@ -60,8 +61,8 @@ def get_matrix_B(d, p, eigmax=1):
 
 
 def generate_varp_process(d=3, n=10000, p=1):
-    rng = np.random.default_rng(42)
-    seed = rng.integers(np.iinfo(np.int32).max)
+    rng = sku.check_random_state(42)
+    seed = rng.randint(np.iinfo(np.int32).max)
     random_state = np.random.RandomState(seed=seed)
     B = get_matrix_B(d, p, eigmax=0.9)
     nu = B[0, :]
@@ -434,7 +435,7 @@ def test_gradient_X_loglik(em: em_sampler.EM, p: int):
     d = 3
     X, _, _, _ = generate_varp_process(d=d, n=10, p=p)
     em.fit_parameters(X)
-    rng = np.random.default_rng(42)
+    rng = sku.check_random_state(42)
     X0 = rng.uniform(0, 10, size=X.shape)
     # X0 = X
     loglik = em.get_loglikelihood(X0)