Merge pull request #30 from Quantmetry/dev_test_cross_validation

Test des fonctions de Cross_Validation

Author: JulienRoussel77

.gitignore

@@ -62,4 +62,6 @@ examples/*.ipynb
 
 # VSCode
 .vscode
-# examples/benchmark.ipynb
+
+# Logs
+nohup.txt

examples/benchmark.md

@@ -73,15 +73,15 @@ cols_to_impute = ["TEMP", "PRES"]
 
 The dataset `Artificial` is designed to have a sum of a periodical signal, a white noise and some outliers.
 
-```python tags=[]
-df_data
-```
-
 ```python
 # df_data = data.get_data_corrupted("Artificial", ratio_masked=.2, mean_size=10)
 # cols_to_impute = ["signal"]
 ```
 
+```python tags=[]
+df_data
+```
+
 Let's take a look at variables to impute. We only consider a station, Aotizhongxin.
 Time series display seasonalities (roughly 12 months).
 
@@ -164,7 +164,7 @@ dict_imputers = {
 }
 n_imputers = len(dict_imputers)
 
-search_params = {
+dict_config_opti = {
     "RPCA_opti": {
         "tau": {"min": .5, "max": 5, "type":"Real"},
         "lam": {"min": .1, "max": 1, "type":"Real"},
@@ -195,7 +195,7 @@ comparison = comparator.Comparator(
     generator_holes = generator_holes,
     metrics=["mae", "wmape", "KL_columnwise", "ks_test", "energy"],
     n_calls_opt=10,
-    search_params=search_params,
+    dict_config_opti=dict_config_opti,
 )
 results = comparison.compare(df_data)
 results
@@ -343,7 +343,7 @@ comparison = comparator.Comparator(
     df_data.columns,
     generator_holes = generator_holes,
     n_calls_opt=10,
-    search_params=search_params,
+    dict_config_opti=dict_config_opti,
 )
 results = comparison.compare(df_data)
 results

nohup.txt

@@ -1,6 +1,5 @@
-import logging
 from functools import partial
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Callable, Dict, List, Optional
 
 import numpy as np
 import pandas as pd
@@ -21,15 +20,15 @@ class Comparator:
         list of column's names selected (all with at least one null value will be imputed)
     columnwise_evaluation : Optional[bool], optional
         whether the metric should be calculated column-wise or not, by default False
-    search_params: Optional[Dict[str, Dict[str, Union[str, float, int]]]] = {}
+    dict_config_opti: Optional[Dict[str, Dict[str, Union[str, float, int]]]] = {}
         dictionary of search space for each implementation method. By default, the value is set to
         {}.
     n_calls_opt: int = 10
         number of calls of the optimization algorithm
         10.
     """
 
-    dict_metrics: Dict[str, Any] = {
+    dict_metrics: Dict[str, Callable] = {
         "mse": metrics.mean_squared_error,
         "rmse": metrics.root_mean_squared_error,
         "mae": metrics.mean_absolute_error,
@@ -50,14 +49,14 @@ def __init__(
         selected_columns: List[str],
         generator_holes: _HoleGenerator,
         metrics: List = ["mae", "wmape", "KL_columnwise"],
-        search_params: Optional[Dict[str, Dict[str, Union[float, int, str]]]] = {},
+        dict_config_opti: Optional[Dict[str, Any]] = {},
         n_calls_opt: int = 10,
     ):
         self.dict_imputers = dict_models
         self.selected_columns = selected_columns
         self.generator_holes = generator_holes
         self.metrics = metrics
-        self.search_params = search_params
+        self.dict_config_opti = dict_config_opti
         self.n_calls_opt = n_calls_opt
 
     def get_errors(
@@ -92,7 +91,7 @@ def evaluate_errors_sample(
         self,
         imputer: Any,
         df: pd.DataFrame,
-        list_spaces: List[Dict] = [],
+        dict_config_opti_imputer: Dict[str, Any] = {},
     ) -> pd.Series:
         """Evaluate the errors in the cross-validation
 
@@ -102,7 +101,7 @@ def evaluate_errors_sample(
             imputation model
         df : pd.DataFrame
             dataframe to impute
-        search_space : Dict
+        dict_config_opti_imputer : Dict
             search space for tested_model's hyperparameters
 
         Returns
@@ -115,12 +114,10 @@ def evaluate_errors_sample(
         for df_mask in self.generator_holes.split(df_origin):
             df_corrupted = df_origin.copy()
             df_corrupted[df_mask] = np.nan
-
-            assert not np.logical_and(df_mask, df_origin.isna()).any().any()
-            if list_spaces:
+            if dict_config_opti_imputer:
                 cv = cross_validation.CrossValidation(
                     imputer,
-                    list_spaces=list_spaces,
+                    dict_config_opti_imputer=dict_config_opti_imputer,
                     hole_generator=self.generator_holes,
                     n_calls=self.n_calls_opt,
                 )
@@ -155,12 +152,12 @@ def compare(
         dict_errors = {}
 
         for name, imputer in self.dict_imputers.items():
-            search_params = self.search_params.get(name, {})
-
-            list_spaces = utils.get_search_space(search_params)
+            dict_config_opti_imputer = self.dict_config_opti.get(name, {})
 
             try:
-                dict_errors[name] = self.evaluate_errors_sample(imputer, df, list_spaces)
+                dict_errors[name] = self.evaluate_errors_sample(
+                    imputer, df, dict_config_opti_imputer
+                )
                 print(f"Tested model: {type(imputer).__name__}")
             except Exception as excp:
                 print("Error while testing ", type(imputer).__name__)