feat: ready to merge

Author: anhkhoangoho

qolmat/benchmark/comparator.py

@@ -165,111 +165,3 @@ def compare(
         df_errors = pd.DataFrame(dict_errors)
 
         return df_errors
-
-
-class ComparatorBasedPattern(Comparator):
-    def __init__(
-        self,
-        dict_models: Dict[str, Any],
-        selected_columns: List[str],
-        generator_holes: _HoleGenerator,
-        metrics: List = ["mae", "wmape", "KL"],
-        search_params: Optional[Dict[str, Dict[str, Union[float, int, str]]]] = {},
-        n_calls_opt: int = 10,
-        num_patterns: int = 5,
-    ):
-        super().__init__(
-            dict_models=dict_models,
-            selected_columns=selected_columns,
-            generator_holes=generator_holes,
-            metrics=metrics,
-            search_params=search_params,
-            n_calls_opt=n_calls_opt,
-        )
-
-        self.num_patterns = num_patterns
-
-    def evaluate_errors_sample(
-        self,
-        imputer: Any,
-        df: pd.DataFrame,
-        list_spaces: List[Dict] = [],
-    ) -> pd.Series:
-        """Evaluate the errors in the cross-validation
-
-        Parameters
-        ----------
-        tested_model : any
-            imputation model
-        df : pd.DataFrame
-            dataframe to impute
-        search_space : Dict
-            search space for tested_model's hyperparameters
-
-        Returns
-        -------
-        pd.DataFrame
-            DataFrame with the errors for each metric (in column) and at each fold (in index)
-        """
-
-        list_errors = []
-        df_origin = df[self.selected_columns].copy()
-        dfs_pattern = self.get_df_based_pattern(df_origin)
-        weights = []
-        # Fit then split, or fit in split ?
-        self.generator_holes.fit(df_origin)
-        for df_pattern in dfs_pattern:
-            # Get all columns in pattern
-            cols_pattern = df_pattern.dropna(axis=1).columns
-            for df_mask in self.generator_holes.split(df_pattern):
-                weights.append(len(df_pattern))
-                df_corrupted = df_pattern.copy()
-                df_corrupted[df_mask] = np.nan
-
-                if list_spaces:
-                    cv = cross_validation.CrossValidation(
-                        imputer,
-                        list_spaces=list_spaces,
-                        hole_generator=self.generator_holes,
-                        n_calls=self.n_calls_opt,
-                    )
-                    df_imputed = cv.fit_transform(df_corrupted)
-                else:
-                    df_imputed = imputer.fit_transform(df_corrupted)
-
-                subset = self.generator_holes.subset  # columns selected
-                subset = [col for col in subset if col in cols_pattern]
-                errors = self.get_errors(df_pattern[subset], df_imputed[subset], df_mask[subset])
-                list_errors.append(errors)
-
-        df_errors = pd.DataFrame(list_errors)
-        # Weighted errors
-        errors_mean = df_errors.apply(
-            lambda x: (x * np.array(weights)).sum() / np.sum(weights), axis=0
-        )
-        return errors_mean.sort_index()
-
-    def get_df_based_pattern(self, df: pd.DataFrame) -> List[pd.DataFrame]:
-        def get_pattern(row):
-            list_col_pattern = [col for col in row.index.to_list() if row[col] == True]
-            if len(list_col_pattern) == 0:
-                return "_EMPTY_"
-            elif len(list_col_pattern) == row.index.size:
-                return "_ALLNAN_"
-            else:
-                return "_".join(list_col_pattern)
-
-        df_isna = df.isna().apply(lambda x: get_pattern(x), axis=1).to_frame(name="pattern")
-        df_isna_pattern = df_isna["pattern"].value_counts()
-
-        patterns = df_isna_pattern.index.to_list()
-        patterns.remove("_ALLNAN_")
-        patterns.remove("_EMPTY_")
-
-        dfs = []
-        for idx_pattern in range(min(len(patterns), self.num_patterns)):
-            patterns_selected = ["_EMPTY_"] + [patterns[idx_pattern]]
-            df_pattern = df.loc[df_isna[df_isna["pattern"].isin(patterns_selected)].index]
-            dfs.append(df_pattern)
-
-        return dfs

qolmat/benchmark/missing_patterns.py

@@ -111,7 +111,7 @@ def split(self, X: pd.DataFrame) -> List[pd.DataFrame]:
             mask
         """
 
-        # self.fit(X)
+        self.fit(X)
         list_masks = []
         for _ in range(self.n_splits):
             if self.ngroups is None: