tests patched, skopt deprecated

Author: Julien Roussel

environment.ci.yml

@@ -6,12 +6,13 @@ dependencies:
     - pip=23.0.1
     - pip:
           - codecov
-          - flake8==6.0.0
-          - matplotlib==3.6.2
-          - mypy==1.1.1
-          - numpydoc==1.5.0
-          - pytest==7.2.0
-          - pytest-cov==4.0.0
-          - pytest-mock==3.10.0
+          - flake8
+          - matplotlib
+          - mypy
+          - numpy==1.19
+          - numpydoc
+          - pytest
+          - pytest-cov
+          - pytest-mock
           - tensorflow
           - -e .

environment.dev.yml

@@ -5,13 +5,14 @@ channels:
 dependencies:
     - bump2version=1.0.1
     - dcor=0.6
-    - ipykernel=5.1.4
+    - ipykernel=6.21.0
     - jupyter=1.0.0
     - jupyterlab=1.2.6
     - jupytext=1.14.4
     - numpy=1.21
     - packaging=23.1
     - pandas=2.0.1
+    - python=3.8
     - pip=23.0.1
     - scipy=1.10.1
     - scikit-learn=1.2.2

examples/benchmark.md

@@ -8,9 +8,9 @@ jupyter:
       format_version: '1.3'
       jupytext_version: 1.14.5
   kernelspec:
-    display_name: Python 3 (ipykernel)
+    display_name: env_qolmat_dev
     language: python
-    name: python3
+    name: env_qolmat_dev
 ---
 
 **This notebook aims to present the Qolmat repo through an example of a multivariate time series.
@@ -62,24 +62,24 @@ The dataset `Beijing` is the Beijing Multi-Site Air-Quality Data Set. It consist
 This dataset only contains numerical vairables.
 
 ```python
-# df_data = data.get_data_corrupted("Beijing", ratio_masked=.2, mean_size=120)
+df_data = data.get_data_corrupted("Beijing", ratio_masked=.2, mean_size=120)
 
 # cols_to_impute = ["TEMP", "PRES", "DEWP", "NO2", "CO", "O3", "WSPM"]
 # cols_to_impute = df_data.columns[df_data.isna().any()]
-# cols_to_impute = ["TEMP", "PRES"]
+cols_to_impute = ["TEMP", "PRES"]
 
 ```
 
 The dataset `Artificial` is designed to have a sum of a periodical signal, a white noise and some outliers.
 
 ```python
-df_data = data.get_data_corrupted("Artificial", ratio_masked=.2, mean_size=10)
-cols_to_impute = ["signal"]
+# df_data = data.get_data_corrupted("Artificial", ratio_masked=.2, mean_size=10)
+# cols_to_impute = ["signal"]
 ```
 
 ```python
-df_data = data.get_data("SNCF", n_groups_max=2)
-cols_to_impute = ["val_in"]
+# df_data = data.get_data("SNCF", n_groups_max=2)
+# cols_to_impute = ["val_in"]
 ```
 
 ```python
@@ -132,14 +132,14 @@ imputer_nocb = imputers.ImputerNOCB(groups=["station"])
 imputer_interpol = imputers.ImputerInterpolation(groups=["station"], method="linear")
 imputer_spline = imputers.ImputerInterpolation(groups=["station"], method="spline", order=2)
 imputer_shuffle = imputers.ImputerShuffle(groups=["station"])
-imputer_residuals = imputers.ImputerResiduals(groups=["station"], period=7, model_tsa="additive", extrapolate_trend="freq", method_interpolation="linear")
+imputer_residuals = imputers.ImputerResiduals(groups=["station"], period=365, model_tsa="additive", extrapolate_trend="freq", method_interpolation="linear")
 
-imputer_rpca = imputers.ImputerRPCA(groups=["station"], columnwise=True, period=7, max_iter=1000, tau=2, lam=1)
+imputer_rpca = imputers.ImputerRPCA(groups=["station"], columnwise=False, max_iter=256, tau=2, lam=1)
 # imputer_rpca_opti = imputers.ImputerRPCA(groups=["station"], columnwise=True, period=7, max_iter=100)
 
 imputer_ou = imputers.ImputerEM(groups=["station"], model="multinormal", method="sample", max_iter_em=34, n_iter_ou=15, dt=1e-3)
 imputer_tsou = imputers.ImputerEM(groups=["station"], model="VAR1", method="sample", max_iter_em=34, n_iter_ou=15, dt=1e-3)
-imputer_tsmle = imputers.ImputerEM(groups=["station"], model="VAR1", method="mle", max_iter_em=100, n_iter_ou=15, dt=1e-3, period=7)
+imputer_tsmle = imputers.ImputerEM(groups=["station"], model="VAR1", method="mle", max_iter_em=100, n_iter_ou=15, dt=1e-3)
 
 
 imputer_knn = imputers.ImputerKNN(groups=["station"], k=10)
@@ -155,7 +155,7 @@ dict_imputers = {
     "shuffle": imputer_shuffle,
     # "residuals": imputer_residuals,
     # "OU": imputer_ou,
-    # "TSOU": imputer_tsou,
+    "TSOU": imputer_tsou,
     "TSMLE": imputer_tsmle,
     "RPCA": imputer_rpca,
     # "RPCA_opti": imputer_rpca_opti,
@@ -184,9 +184,6 @@ In order to compare the methods, we $i)$ artificially create missing data (for m
 </p>
 
 
-```python
-imputer_tsmle.hyperparams_user
-```
 
 Concretely, the comparator takes as input a dataframe to impute, a proportion of nan to create, a dictionary of imputers (those previously mentioned), a list with the columns names to impute, a generator of holes specifying the type of holes to create and the search dictionary search_params for hyperparameter optimization.
 

qolmat/benchmark/comparator.py

@@ -64,7 +64,7 @@ def get_errors(
         df_origin: pd.DataFrame,
         df_imputed: pd.DataFrame,
         df_mask: pd.DataFrame,
-    ) -> pd.DataFrame:
+    ) -> pd.Series:
         """Functions evaluating the reconstruction's quality
 
         Parameters

qolmat/benchmark/cross_validation.py

@@ -213,7 +213,7 @@ def optimize_hyperparams(self, df: pd.DataFrame) -> Dict[str, Any]:
         Parameters
         ----------
         df : pd.DataFrame
-            DataFrame masked
+            DataFrame with nans
 
         Returns
         -------

qolmat/imputations/em_sampler.py

@@ -165,7 +165,7 @@ def __init__(
         tolerance: float = 1e-4,
         stagnation_threshold: float = 5e-3,
         stagnation_loglik: float = 2,
-        period: Optional[int] = None,
+        period: int = 1,
     ):
         if method not in ["mle", "sample"]:
             raise ValueError(f"`method` must be 'mle' or 'sample', provided value is '{method}'")
@@ -223,8 +223,8 @@ def fit(self, X: NDArray):
         if not isinstance(X, np.ndarray):
             raise AssertionError("Invalid type. X must be a NDArray.")
 
-        X = self.scaler.fit_transform(X.T).T
         X = utils.prepare_data(X, self.period)
+        X = self.scaler.fit_transform(X.T).T
 
         mask_na = np.isnan(X)
 
@@ -332,7 +332,7 @@ def __init__(
         tolerance: float = 1e-4,
         stagnation_threshold: float = 5e-3,
         stagnation_loglik: float = 2,
-        period: Optional[int] = None,
+        period: int = 1,
     ) -> None:
         super().__init__(
             method=method,
@@ -545,7 +545,7 @@ def __init__(
         tolerance: float = 1e-4,
         stagnation_threshold: float = 5e-3,
         stagnation_loglik: float = 2,
-        period: Optional[int] = None,
+        period: int = 1,
     ) -> None:
         super().__init__(
             method=method,

qolmat/imputations/rpca/rpca.py

@@ -31,7 +31,7 @@ class RPCA(BaseEstimator, TransformerMixin):
 
     def __init__(
         self,
-        period: Optional[int] = None,
+        period: int = 1,
         max_iter: int = int(1e4),
         tol: float = 1e-6,
         random_state: Union[None, int, np.random.RandomState] = None,
@@ -60,20 +60,13 @@ def decompose_rpca_signal(
         A: NDArray
             Anomalies
         """
-        D_init = utils.prepare_data(X, self.period)
-        Omega = ~np.isnan(D_init)
+        D = utils.prepare_data(X, self.period)
+        Omega = ~np.isnan(D)
         # D_proj = rpca_utils.impute_nans(D_init, method="median")
-        D_proj = D_init.T
-        D_proj = utils.linear_interpolation(D_proj)
+        D = utils.linear_interpolation(D)
 
-        # self.scaler = StandardScaler()
-        # D_proj = self.scaler.fit_transform(D_proj)
-        D_proj = D_proj.T
+        M, A = self.decompose_rpca(D, Omega)
 
-        M, A = self.decompose_rpca(D_proj, Omega)
-
-        # M = self.scaler.inverse_transform(M.T).T
-        # A = self.scaler.inverse_transform(A.T).T
         M_final = utils.get_shape_original(M, X.shape)
         A_final = utils.get_shape_original(A, X.shape)
 

qolmat/imputations/rpca/rpca_noisy.py

@@ -51,7 +51,7 @@ class RPCANoisy(RPCA):
 
     def __init__(
         self,
-        period: Optional[int] = None,
+        period: int = 1,
         rank: Optional[int] = None,
         tau: Optional[float] = None,
         lam: Optional[float] = None,

qolmat/imputations/rpca/rpca_pcp.py

@@ -29,7 +29,7 @@ class RPCAPCP(RPCA):
 
     def __init__(
         self,
-        period: Optional[int] = None,
+        period: int = 1,
         mu: Optional[float] = None,
         lam: Optional[float] = None,
         max_iter: int = int(1e4),

qolmat/utils/exceptions.py

@@ -4,3 +4,11 @@ def __init__(self):
             """Please install keras xx.xx.xx
         pip install qolmat[keras]"""
         )
+
+
+class SignalTooShort(Exception):
+    def __init__(self, period, n_cols):
+        super().__init__(
+            f"""`period` must be smaller than the signals duration.
+            `period`is {period} but the number of columns if {n_cols}"""
+        )