[API 2]: CFI, PFI, LOCO

examples/plot_conditional_vs_marginal_xor_data.py

@@ -117,7 +117,7 @@
         random_state=0,
     )
     vim.fit(X_train, y_train)
-    importances.append(vim.importance(X_test, y_test)["importance"])
+    importances.append(vim.importance(X_test, y_test))
 
 importances = np.array(importances).T
 

examples/plot_diabetes_variable_importance_example.py

@@ -184,14 +184,14 @@ def compute_pval(vim):
 # -------------------
 
 
-cfi_vim_arr = np.array([x["importance"] for x in cfi_importance_list]) / 2
+cfi_vim_arr = np.array(cfi_importance_list) / 2
 cfi_pval = compute_pval(cfi_vim_arr)
 
 vim = [
     pd.DataFrame(
         {
             "var": np.arange(cfi_vim_arr.shape[1]),
-            "importance": x["importance"],
+            "importance": x,
             "fold": i,
             "pval": cfi_pval,
             "method": "CFI",
@@ -200,14 +200,14 @@ def compute_pval(vim):
     for x in cfi_importance_list
 ]
 
-loco_vim_arr = np.array([x["importance"] for x in loco_importance_list])
+loco_vim_arr = np.array(loco_importance_list)
 loco_pval = compute_pval(loco_vim_arr)
 
 vim += [
     pd.DataFrame(
         {
             "var": np.arange(loco_vim_arr.shape[1]),
-            "importance": x["importance"],
+            "importance": x,
             "fold": i,
             "pval": loco_pval,
             "method": "LOCO",
@@ -216,14 +216,14 @@ def compute_pval(vim):
     for x in loco_importance_list
 ]
 
-pfi_vim_arr = np.array([x["importance"] for x in pfi_importance_list])
+pfi_vim_arr = np.array(pfi_importance_list)
 pfi_pval = compute_pval(pfi_vim_arr)
 
 vim += [
     pd.DataFrame(
         {
             "var": np.arange(pfi_vim_arr.shape[1]),
-            "importance": x["importance"],
+            "importance": x,
             "fold": i,
             "pval": pfi_pval,
             "method": "PFI",

examples/plot_importance_classification_iris.py

@@ -93,7 +93,7 @@ def run_one_fold(X, y, model, train_index, test_index, vim_name="CFI", groups=No
         )
 
     vim.fit(X[train_index], y[train_index], groups=groups)
-    importance = vim.importance(X[test_index], y[test_index])["importance"]
+    importance = vim.importance(X[test_index], y[test_index])
 
     return pd.DataFrame(
         {

examples/plot_model_agnostic_importance.py

@@ -108,10 +108,8 @@
     vim_linear.fit(X[train], y[train])
     vim_non_linear.fit(X[train], y[train])
 
-    importances_linear.append(vim_linear.importance(X[test], y[test])["importance"])
-    importances_non_linear.append(
-        vim_non_linear.importance(X[test], y[test])["importance"]
-    )
+    importances_linear.append(vim_linear.importance(X[test], y[test]))
+    importances_non_linear.append(vim_non_linear.importance(X[test], y[test]))
 
 
 ################################################################################

examples/plot_pitfalls_permutation_importance.py

@@ -132,7 +132,7 @@
     )
     pfi.fit(X_test, y_test)
 
-    permutation_importances.append(pfi.importance(X_test, y_test)["importance"])
+    permutation_importances.append(pfi.importance(X_test, y_test))
 permutation_importances = np.stack(permutation_importances)
 pval_pfi = ttest_1samp(
     permutation_importances, 0.0, axis=0, alternative="greater"
@@ -200,7 +200,7 @@
     )
     cfi.fit(X_test, y_test)
 
-    conditional_importances.append(cfi.importance(X_test, y_test)["importance"])
+    conditional_importances.append(cfi.importance(X_test, y_test))
 
 
 cfi_pval = ttest_1samp(

src/hidimstat/__init__.py

@@ -14,16 +14,16 @@
     desparsified_group_lasso_pvalue,
 )
 from .distilled_conditional_randomization_test import d0crt, D0CRT
-from .conditional_feature_importance import CFI
+from .conditional_feature_importance import cfi, CFI
 from .knockoffs import (
     model_x_knockoff,
     model_x_knockoff_pvalue,
     model_x_knockoff_bootstrap_quantile,
     model_x_knockoff_bootstrap_e_value,
 )
-from .leave_one_covariate_out import LOCO
+from .leave_one_covariate_out import loco, LOCO
 from .noise_std import reid
-from .permutation_feature_importance import PFI
+from .permutation_feature_importance import pfi, PFI
 
 from .statistical_tools.aggregation import quantile_aggregation
 
@@ -49,6 +49,9 @@
     "model_x_knockoff_bootstrap_quantile",
     "model_x_knockoff_bootstrap_e_value",
     "CFI",
+    "cfi",
     "LOCO",
+    "loco",
     "PFI",
+    "pfi",
 ]

src/hidimstat/base_perturbation.py

@@ -1,9 +1,12 @@
+import warnings
+
 import numpy as np
 import pandas as pd
 from joblib import Parallel, delayed
-from sklearn.base import check_is_fitted
+from scipy.stats import ttest_1samp
+from sklearn.base import check_is_fitted, clone
 from sklearn.metrics import root_mean_squared_error
-import warnings
+from sklearn.model_selection import KFold
 
 from hidimstat._utils.utils import _check_vim_predict_method
 from hidimstat._utils.exception import InternalError
@@ -14,9 +17,9 @@ class BasePerturbation(BaseVariableImportance):
     def __init__(
         self,
         estimator,
+        method: str = "predict",
         loss: callable = root_mean_squared_error,
         n_permutations: int = 50,
-        method: str = "predict",
         n_jobs: int = 1,
     ):
         """
@@ -27,6 +30,10 @@ def __init__(
         ----------
         estimator : sklearn compatible estimator, optional
             The estimator to use for the prediction.
+        method : str, default="predict"
+            The method used for making predictions. This determines the predictions
+            passed to the loss function. Supported methods are "predict",
+            "predict_proba", "decision_function", "transform".
         loss : callable, default=root_mean_squared_error
             The function to compute the loss when comparing the perturbed model
             to the original model.
@@ -35,10 +42,6 @@ def __init__(
             Specifies the number of times the variable group (residual for CFI) is
             permuted. For each permutation, the perturbed model's loss is calculated
             and averaged over all permutations.
-        method : str, default="predict"
-            The method used for making predictions. This determines the predictions
-            passed to the loss function. Supported methods are "predict",
-            "predict_proba", "decision_function", "transform".
         n_jobs : int, default=1
             The number of parallel jobs to run. Parallelization is done over the
             variables or groups of variables.
@@ -50,9 +53,18 @@ def __init__(
         self.loss = loss
         _check_vim_predict_method(method)
         self.method = method
-        self.n_jobs = n_jobs
         self.n_permutations = n_permutations
-        self.n_groups = None
+        self.n_jobs = n_jobs
+        # variable set in fit
+        self.groups = None
+        # varaible set in importance
+        self.loss_reference_ = None
+        self.loss_ = None
+        # variable set in fit_importance
+        self.importances_cv_ = None
+        # internal variables
+        self._n_groups = None
+        self._groups_ids = None
 
     def fit(self, X, y=None, groups=None):
         """Base fit method for perturbation-based methods. Identifies the groups.
@@ -69,11 +81,11 @@ def fit(self, X, y=None, groups=None):
             identified based on the columns of X.
         """
         if groups is None:
-            self.n_groups = X.shape[1]
-            self.groups = {j: [j] for j in range(self.n_groups)}
+            self._n_groups = X.shape[1]
+            self.groups = {j: [j] for j in range(self._n_groups)}
             self._groups_ids = np.array(list(self.groups.values()), dtype=int)
         elif isinstance(groups, dict):
-            self.n_groups = len(groups)
+            self._n_groups = len(groups)
             self.groups = groups
             if isinstance(X, pd.DataFrame):
                 self._groups_ids = []
@@ -91,6 +103,7 @@ def fit(self, X, y=None, groups=None):
                 ]
         else:
             raise ValueError("groups needs to be a dictionnary")
+        return self
 
     def predict(self, X):
         """
@@ -139,27 +152,69 @@ def importance(self, X, y):
         """
         self._check_fit(X)
 
-        out_dict = dict()
-
         y_pred = getattr(self.estimator, self.method)(X)
-        loss_reference = self.loss(y, y_pred)
-        out_dict["loss_reference"] = loss_reference
+        self.loss_reference_ = self.loss(y, y_pred)
 
         y_pred = self.predict(X)
-        out_dict["loss"] = dict()
+        self.loss_ = dict()
         for j, y_pred_j in enumerate(y_pred):
             list_loss = []
             for y_pred_perm in y_pred_j:
                 list_loss.append(self.loss(y, y_pred_perm))
-            out_dict["loss"][j] = np.array(list_loss)
+            self.loss_[j] = np.array(list_loss)
 
-        out_dict["importance"] = np.array(
+        self.importances_ = np.array(
             [
-                np.mean(out_dict["loss"][j]) - loss_reference
-                for j in range(self.n_groups)
+                np.mean(self.loss_[j]) - self.loss_reference_
+                for j in range(self._n_groups)
             ]
         )
-        return out_dict
+        self.pvalues_ = ttest_1samp(
+            self.importances_, 0.0, axis=0, alternative="greater"
+        ).pvalue
+        return self.importances_
+
+    def fit_importance(
+        self, X, y, cv=KFold(n_splits=5, shuffle=True, random_state=0), **fit_kwargs
+    ):
+        """
+        Compute feature importance scores using cross-validation.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            Training data.
+        y : array-like of shape (n_samples,)
+            Target values.
+        cv : cross-validation generator or iterable, default=KFold(n_splits=5, shuffle=True, random_state=0)
+            Determines the cross-validation splitting strategy.
+        **fit_kwargs : dict
+            Additional arguments passed to the fit method during variable group identification.
+
+        Returns
+        -------
+        importances : float
+            Mean feature importance scores across CV folds.
+
+        Notes
+        -----
+        For each CV fold:
+        1. Clones and fits the estimator on training fold
+        2. Identifies variable groups on training fold
+        3. Computes feature importances on test fold
+        4. Returns average importance across all folds
+
+        The importances for each fold are stored in self.importances\_
+        """
+        importances = []
+        for train, test in cv.split(X):
+            estimator = clone(self.estimator)
+            estimator.fit(X[train], y[train])
+            self.fit(X[train], y[train], **fit_kwargs)
+            importances.append(self.importance(X[test], y[test]))
+        self.importances_cv_ = importances
+        self.importances_ = np.mean(importances, axis=0)
+        return self.importances_
 
     def _check_fit(self, X):
         """
@@ -183,11 +238,7 @@ def _check_fit(self, X):
             If the number of features in X does not match the total number
             of features in the grouped variables.
         """
-        if (
-            self.n_groups is None
-            or not hasattr(self, "groups")
-            or not hasattr(self, "_groups_ids")
-        ):
+        if self._n_groups is None or self.groups is None or self._groups_ids is None:
             raise ValueError(
                 "The class is not fitted. The fit method must be called"
                 " to set variable groups. If no grouping is needed,"
@@ -231,6 +282,16 @@ def _check_fit(self, X):
                 f"{number_unique_feature_in_groups}"
             )
 
+    def _check_importance(self):
+        """
+        Checks if the loss have been computed.
+        """
+        super()._check_importance()
+        if self.loss_reference_ is None or self.loss_ is None:
+            raise ValueError(
+                "The importances need to be called before calling this method"
+            )
+
     def _joblib_predict_one_group(self, X, group_id, group_key):
         """
         Compute the predictions after perturbation of the data for a given