Merge pull request #173 from scikit-learn-contrib/feature/holes_reproduction

Feature/holes reproduction

Author: JulienRoussel77

examples/tutorials/plot_tuto_benchmark_TS.py

@@ -25,6 +25,10 @@
 from qolmat.benchmark import comparator, missing_patterns
 from qolmat.imputations import imputers
 from qolmat.utils import data, plot
+from sklearn import utils as sku
+
+seed = 1234
+rng = sku.check_random_state(seed)
 
 # %%
 # 1. Data
@@ -96,6 +100,7 @@
     n_iter_ou=15,
     dt=1e-3,
     p=1,
+    random_state=rng
 )
 imputer_mice = imputers.ImputerMICE(
     groups=("station",),
@@ -109,6 +114,7 @@
     groups=("station",),
     subset=cols_to_impute,
     ratio_masked=ratio_masked,
+    random_state=rng
 )
 
 dict_imputers = {

examples/tutorials/plot_tuto_categorical.py

@@ -8,12 +8,16 @@
 """
 
 from sklearn.pipeline import Pipeline
+from sklearn import utils as sku
 
 from qolmat.benchmark import comparator, missing_patterns
 from qolmat.imputations import imputers, preprocessing
 from qolmat.imputations.imputers import ImputerRegressor
 from qolmat.utils import data
 
+seed = 1234
+rng = sku.check_random_state(seed)
+
 # %%
 # 1. Titanic dataset
 # ---------------------------------------------------------------
@@ -39,7 +43,7 @@
 
 cols_num = df.select_dtypes(include="number").columns
 cols_cat = df.select_dtypes(exclude="number").columns
-imputer_rpca = imputers.ImputerRpcaNoisy()
+imputer_rpca = imputers.ImputerRpcaNoisy(random_state=rng)
 ohe = preprocessing.OneHotEncoderProjector(
     handle_unknown="ignore",
     handle_missing="return_nan",
@@ -58,7 +62,7 @@
 # - manage missing features (native to the HistGradientBoosting)
 
 pipestimator = preprocessing.make_robust_MixteHGB(avoid_new=True)
-imputer_hgb = ImputerRegressor(estimator=pipestimator, handler_nan="none")
+imputer_hgb = ImputerRegressor(estimator=pipestimator, handler_nan="none", random_state=rng)
 imputer_wrap_hgb = preprocessing.WrapperTransformer(imputer_hgb, bt)
 
 # %%
@@ -79,6 +83,7 @@
     subset=cols_to_impute,
     ratio_masked=ratio_masked,
     sample_proportional=False,
+    random_state=rng
 )
 metrics = ["rmse", "accuracy"]
 

examples/tutorials/plot_tuto_diffusion_models.py

@@ -9,12 +9,15 @@
 import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
+from sklearn import utils as sku
 
 from qolmat.benchmark import comparator, missing_patterns
-from qolmat.imputations.diffusions.ddpms import TabDDPM, TsDDPM
 from qolmat.imputations.imputers_pytorch import ImputerDiffusion
 from qolmat.utils import data
 
+seed = 1234
+rng = sku.check_random_state(seed)
+
 logging.basicConfig(
     format="%(asctime)s %(levelname)-8s %(message)s",
     level=logging.INFO,
@@ -31,7 +34,7 @@
 # For this tutorial, we only use a small subset of this data
 # 1000 rows and 2 features (TEMP, PRES).
 
-df_data = data.get_data_corrupted("Beijing")
+df_data = data.get_data_corrupted("Beijing", random_state=rng)
 df_data = df_data[["TEMP", "PRES"]].iloc[:1000]
 df_data.index = df_data.index.set_levels(
     [df_data.index.levels[0], pd.to_datetime(df_data.index.levels[1])]
@@ -75,6 +78,7 @@
     batch_size=100,
     x_valid=df_data_valid,
     print_valid=True,
+    random_state=rng,
 )
 tabddpm = tabddpm.fit(df_data)
 
@@ -159,19 +163,19 @@
 # reconstruction errors (mae) but increases distribution distance (kl_columnwise).
 
 dict_imputers = {
-    "num_sampling=5": ImputerDiffusion(epochs=10, batch_size=100, num_sampling=5),
-    "num_sampling=10": ImputerDiffusion(epochs=10, batch_size=100, num_sampling=10),
+    "num_sampling=5": ImputerDiffusion(epochs=10, batch_size=100, num_sampling=5, random_state=rng),
+    "num_sampling=10": ImputerDiffusion(epochs=10, batch_size=100, num_sampling=10, random_state=rng),
 }
 
 comparison = comparator.Comparator(
     dict_imputers,
     selected_columns=df_data.columns,
-    generator_holes=missing_patterns.UniformHoleGenerator(n_splits=2),
+    generator_holes=missing_patterns.UniformHoleGenerator(n_splits=2, random_state=rng),
     metrics=["mae", "kl_columnwise"],
 )
 results = comparison.compare(df_data)
 
-results.groupby(axis=0, level=0).mean().groupby(axis=0, level=0).mean()
+results.groupby(level=0).mean().groupby(level=0).mean()
 
 # %%
 # 4. Hyperparameters for TsDDPM
@@ -205,7 +209,7 @@
 #   but requires a longer training/inference time.
 
 dict_imputers = {
-    "tabddpm": ImputerDiffusion(model="TabDDPM", epochs=10, batch_size=100, num_sampling=5
+    "tabddpm": ImputerDiffusion(model="TabDDPM", epochs=10, batch_size=100, num_sampling=5, random_state=rng
     ),
     "tsddpm": ImputerDiffusion(
         model="TsDDPM",
@@ -214,19 +218,19 @@
         index_datetime="date",
         freq_str="5D",
         num_sampling=5,
-        is_rolling=False
+        is_rolling=False, random_state=rng
     ),
 }
 
 comparison = comparator.Comparator(
     dict_imputers,
     selected_columns=df_data.columns,
-    generator_holes=missing_patterns.UniformHoleGenerator(n_splits=2),
+    generator_holes=missing_patterns.UniformHoleGenerator(n_splits=2, random_state=rng),
     metrics=["mae", "kl_columnwise"],
 )
 results = comparison.compare(df_data)
 
-results.groupby(axis=0, level=0).mean().groupby(axis=0, level=0).mean()
+results.groupby(level=0).mean().groupby(level=0).mean()
 
 # %%
 # [1] Ho, Jonathan, Ajay Jain, and Pieter Abbeel. `Denoising diffusion probabilistic models.

examples/tutorials/plot_tuto_hole_generator.py

@@ -21,10 +21,15 @@
 import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
+from sklearn import utils as sku
+from torch import rand
 
 from qolmat.benchmark import missing_patterns
 from qolmat.utils import data
 
+seed = 1234
+rng = sku.check_random_state(seed)
+
 # %%
 # 1. Data
 # ---------------------------------------------------------------
@@ -42,7 +47,7 @@
 columns = ["TEMP", "PRES", "DEWP", "RAIN", "WSPM"]
 df_data = df_data[columns]
 
-df = data.add_holes(df_data, ratio_masked=0.2, mean_size=120)
+df = data.add_holes(df_data, ratio_masked=0.2, mean_size=120, random_state=rng)
 cols_to_impute = df.columns
 
 # %%
@@ -169,8 +174,8 @@ def plot_cdf(
         axs[ind].plot(sorted_data, cdf, c="gray", lw=2, label="original")
 
     for df_mask, label, color in zip(list_df_mask, labels, colors):
-        array_mask = df_mask.copy()
-        array_mask[array_mask == True] = np.nan
+        array_mask = df_mask.astype(float).copy()
+        array_mask[df_mask] = np.nan
         hole_sizes_created = get_holes_sizes_column_wise(array_mask.to_numpy())
 
         for ind, (hole_created, col) in enumerate(
@@ -197,7 +202,7 @@ def plot_cdf(
 # Note this class is more suited for tabular datasets.
 
 uniform_generator = missing_patterns.UniformHoleGenerator(
-    n_splits=1, subset=df.columns, ratio_masked=0.1
+    n_splits=1, subset=df.columns, ratio_masked=0.1, random_state=rng
 )
 uniform_mask = uniform_generator.split(df)[0]
 
@@ -223,7 +228,7 @@ def plot_cdf(
 # :class:`~qolmat.benchmark.missing_patterns.UniformHoleGenerator` class.
 
 geometric_generator = missing_patterns.GeometricHoleGenerator(
-    n_splits=1, subset=cols_to_impute, ratio_masked=0.1
+    n_splits=1, subset=cols_to_impute, ratio_masked=0.1, random_state=rng
 )
 geometric_mask = geometric_generator.split(df)[0]
 
@@ -249,7 +254,7 @@ def plot_cdf(
 # is learned on each group: here on each station.
 
 empirical_generator = missing_patterns.EmpiricalHoleGenerator(
-    n_splits=1, subset=df.columns, ratio_masked=0.1, groups=("station",)
+    n_splits=1, subset=df.columns, ratio_masked=0.1, groups=("station",), random_state=rng
 )
 empirical_mask = empirical_generator.split(df)[0]
 
@@ -274,7 +279,7 @@ def plot_cdf(
 # :class:`~qolmat.benchmark.missing_patterns.MultiMarkovHoleGenerator` class.
 
 multi_markov_generator = missing_patterns.MultiMarkovHoleGenerator(
-    n_splits=1, subset=df.columns, ratio_masked=0.1
+    n_splits=1, subset=df.columns, ratio_masked=0.1, random_state=rng
 )
 multi_markov_mask = multi_markov_generator.split(df)[0]
 
@@ -297,7 +302,7 @@ def plot_cdf(
 # :class:`~qolmat.benchmark.missing_patterns.GroupedHoleGenerator` class.
 
 grouped_generator = missing_patterns.GroupedHoleGenerator(
-    n_splits=1, subset=df.columns, ratio_masked=0.1, groups=("station",)
+    n_splits=1, subset=df.columns, ratio_masked=0.1, groups=("station",), random_state=rng
 )
 grouped_mask = grouped_generator.split(df)[0]
 

examples/tutorials/plot_tuto_mcar.py

@@ -13,18 +13,19 @@
 import numpy as np
 import pandas as pd
 from scipy.stats import norm
+from sklearn import utils as sku
 
 from qolmat.analysis.holes_characterization import LittleTest, PKLMTest
 from qolmat.benchmark.missing_patterns import UniformHoleGenerator
 
 plt.rcParams.update({"font.size": 12})
-
+seed = 1234
+rng = sku.check_random_state(seed)
 
 # %%
 # Generating random data
 # ----------------------
 
-rng = np.random.RandomState(42)
 data = rng.multivariate_normal(mean=[0, 0], cov=[[1, 0], [0, 1]], size=200)
 df = pd.DataFrame(data=data, columns=["Column 1", "Column 2"])
 

examples/tutorials/plot_tuto_mean_median.py

@@ -4,7 +4,7 @@
 
 In this tutorial, we show how to use the Qolmat comparator
 (:class:`~qolmat.benchmark.comparator`) to choose
-the best imputation between imputation by the mean or the median
+the best imputation between two of the simplest imputation methods: mean or median
 (:class:`~qolmat.imputations.imputers.ImputerSimple`).
 The dataset used is the the numerical `superconduct` dataset and
 contains information on 21263 superconductors.
@@ -15,11 +15,15 @@
 import matplotlib
 import matplotlib.pyplot as plt
 import numpy as np
+from sklearn import utils as sku
 
 from qolmat.benchmark import comparator, missing_patterns
 from qolmat.imputations import imputers
 from qolmat.utils import data, plot
 
+seed = 1234
+rng = sku.check_random_state(seed)
+
 # %%
 # 1. Data
 # ---------------------------------------------------------------
@@ -33,7 +37,7 @@
 # In this way, each column has missing values.
 
 df = data.add_holes(
-    data.get_data("Superconductor"), ratio_masked=0.2, mean_size=120
+    data.get_data("Superconductor"), ratio_masked=0.2, mean_size=120, random_state=rng
 )
 
 # %%
@@ -91,7 +95,7 @@
 # ``subset=cols_to_impute``:
 
 generator_holes = missing_patterns.UniformHoleGenerator(
-    n_splits=2, subset=cols_to_impute, ratio_masked=0.1
+    n_splits=2, subset=cols_to_impute, ratio_masked=0.1, random_state=rng
 )
 df_mask = generator_holes.generate_mask(df)
 df_mask = np.invert(df_mask).astype("int")

qolmat/analysis/holes_characterization.py

@@ -14,6 +14,7 @@
 
 from qolmat.imputations.imputers import ImputerEM
 from qolmat.utils.input_check import check_pd_df_dtypes
+from qolmat.utils.utils import RandomSetting
 
 
 class McarTest(ABC):
@@ -32,9 +33,7 @@ class McarTest(ABC):
 
     """
 
-    def __init__(
-        self, random_state: Union[None, int, np.random.RandomState] = None
-    ):
+    def __init__(self, random_state: RandomSetting = None):
         """Initialize the McarTest class with a random state.
 
         Parameters
@@ -95,7 +94,7 @@ class LittleTest(McarTest):
     def __init__(
         self,
         imputer: Optional[ImputerEM] = None,
-        random_state: Union[None, int, np.random.RandomState] = None,
+        random_state: RandomSetting = None,
     ):
         super().__init__()
         if imputer and imputer.model != "multinormal":
@@ -203,7 +202,7 @@ def __init__(
         nb_trees_per_proj: int = 200,
         compute_partial_p_values: bool = False,
         encoder: Union[None, OneHotEncoder] = None,
-        random_state: Union[None, int, np.random.RandomState] = None,
+        random_state: RandomSetting = None,
     ):
         super().__init__(random_state=random_state)
         self.nb_projections = nb_projections

qolmat/benchmark/comparator.py

@@ -6,6 +6,7 @@
 import numpy as np
 import pandas as pd
 from joblib import Parallel, cpu_count, delayed
+from sklearn import utils as sku
 
 from qolmat.benchmark import hyperparameters, metrics
 from qolmat.benchmark.missing_patterns import _HoleGenerator
@@ -110,6 +111,12 @@ def process_split(
             errors results
 
         """
+        self.generator_holes.random_state = sku.check_random_state(
+            self.generator_holes.random_state
+        )
+        self.generator_holes.save_rng_state()
+        for name, imputer in self.dict_imputers.items():
+            self.generator_holes.load_rng_state()
         _, df_mask, df_origin = split_data
         df_with_holes = df_origin.copy()
         df_with_holes[df_mask] = np.nan