[API 2]: CFI, PFI, LOCO

docs/src/api.rst

@@ -37,11 +37,14 @@ Feature Importance functions
 .. autosummary::
    :toctree: ./generated/api/class/
    :template: function.rst
-
+
+   cfi_analysis
    clustered_inference
    clustered_inference_pvalue
    ensemble_clustered_inference
    ensemble_clustered_inference_pvalue
+   loco_analysis
+   pfi_analysis
 
 Visualization
 =============
@@ -60,8 +63,8 @@ Samplers
    :toctree: ./generated/api/class/
    :template: class.rst
 
-   ~statistical_tools.ConditionalSampler
-   ~statistical_tools.GaussianKnockoffs
+   ~samplers.ConditionalSampler
+   ~samplers.GaussianKnockoffs
 
 Helper Functions
 ================

examples/plot_conditional_vs_marginal_xor_data.py

@@ -131,7 +131,7 @@
         random_state=0,
     )
     vim.fit(X_train, y_train)
-    importances.append(vim.importance(X_test, y_test)["importance"])
+    importances.append(vim.importance(X_test, y_test))
 
 importances = np.array(importances).T
 

examples/plot_diabetes_variable_importance_example.py

@@ -63,7 +63,7 @@
 import numpy as np
 from sklearn.base import clone
 from sklearn.linear_model import LogisticRegressionCV, RidgeCV
-from sklearn.metrics import r2_score, root_mean_squared_error
+from sklearn.metrics import mean_squared_error, r2_score
 from sklearn.model_selection import KFold
 
 n_folds = 5
@@ -78,7 +78,7 @@
     score = r2_score(
         y_true=y[test_index], y_pred=regressor_list[i].predict(X[test_index])
     )
-    mse = root_mean_squared_error(
+    mse = mean_squared_error(
         y_true=y[test_index], y_pred=regressor_list[i].predict(X[test_index])
     )
 
@@ -166,14 +166,14 @@
 import pandas as pd
 from scipy.stats import ttest_1samp
 
-cfi_vim_arr = np.array([x["importance"] for x in cfi_importance_list]) / 2
+cfi_vim_arr = np.array(cfi_importance_list) / 2
 cfi_pval = ttest_1samp(cfi_vim_arr, 0, alternative="greater").pvalue
 
 vim = [
     pd.DataFrame(
         {
             "var": np.arange(cfi_vim_arr.shape[1]),
-            "importance": x["importance"],
+            "importance": x,
             "fold": i,
             "pval": cfi_pval,
             "method": "CFI",
@@ -182,14 +182,14 @@
     for x in cfi_importance_list
 ]
 
-loco_vim_arr = np.array([x["importance"] for x in loco_importance_list])
+loco_vim_arr = np.array(loco_importance_list)
 loco_pval = ttest_1samp(loco_vim_arr, 0, alternative="greater").pvalue
 
 vim += [
     pd.DataFrame(
         {
             "var": np.arange(loco_vim_arr.shape[1]),
-            "importance": x["importance"],
+            "importance": x,
             "fold": i,
             "pval": loco_pval,
             "method": "LOCO",
@@ -198,14 +198,14 @@
     for x in loco_importance_list
 ]
 
-pfi_vim_arr = np.array([x["importance"] for x in pfi_importance_list])
+pfi_vim_arr = np.array(pfi_importance_list)
 pfi_pval = ttest_1samp(pfi_vim_arr, 0, alternative="greater").pvalue
 
 vim += [
     pd.DataFrame(
         {
             "var": np.arange(pfi_vim_arr.shape[1]),
-            "importance": x["importance"],
+            "importance": x,
             "fold": i,
             "pval": pfi_pval,
             "method": "PFI",

examples/plot_importance_classification_iris.py

@@ -107,7 +107,7 @@ def run_one_fold(
         )
 
     vim.fit(X[train_index], y[train_index])
-    importance = vim.importance(X[test_index], y[test_index])["importance"]
+    importance = vim.importance(X[test_index], y[test_index])
 
     return pd.DataFrame(
         {

examples/plot_knockoffs_wisconsin.py

@@ -147,7 +147,7 @@
 from sklearn.covariance import LedoitWolf
 
 from hidimstat import ModelXKnockoff
-from hidimstat.statistical_tools.gaussian_knockoffs import GaussianKnockoffs
+from hidimstat.samplers import GaussianKnockoffs
 
 model_x_knockoff = ModelXKnockoff(
     ko_generator=GaussianKnockoffs(

examples/plot_loco.py

@@ -82,7 +82,7 @@
     # importance. This process is repeated for all features to assess their individual
     # contributions.
     loco.fit(X_train, y_train)
-    importances = loco.importance(X_test, y_test)["importance"]
+    importances = loco.importance(X_test, y_test)
     df_list.append(
         pd.DataFrame(
             {

examples/plot_model_agnostic_importance.py

@@ -134,12 +134,8 @@
     vim_linear.fit(X[train], y[train])
     vim_non_linear.fit(X[train], y[train])
 
-    importances_linear.append(
-        vim_linear.importance(X[test], y[test])["importance"],
-    )
-    importances_non_linear.append(
-        vim_non_linear.importance(X[test], y[test])["importance"]
-    )
+    importances_linear.append(vim_linear.importance(X[test], y[test]))
+    importances_non_linear.append(vim_non_linear.importance(X[test], y[test]))
 
 
 # %%

examples/plot_pitfalls_permutation_importance.py

@@ -144,7 +144,7 @@
     )
     pfi.fit(X_test, y_test)
 
-    permutation_importances.append(pfi.importance(X_test, y_test)["importance"])
+    permutation_importances.append(pfi.importance(X_test, y_test))
 permutation_importances = np.stack(permutation_importances)
 pval_pfi = ttest_1samp(
     permutation_importances, 0.0, axis=0, alternative="greater"
@@ -216,7 +216,7 @@
     )
     cfi.fit(X_test, y_test)
 
-    conditional_importances.append(cfi.importance(X_test, y_test)["importance"])
+    conditional_importances.append(cfi.importance(X_test, y_test))
 
 
 cfi_pval = ttest_1samp(
@@ -267,7 +267,7 @@
 
 from matplotlib.lines import Line2D
 
-from hidimstat.statistical_tools.conditional_sampling import ConditionalSampler
+from hidimstat.samplers.conditional_sampling import ConditionalSampler
 
 X_train, X_test = train_test_split(
     X,

src/hidimstat/__init__.py

@@ -1,4 +1,4 @@
-from .conditional_feature_importance import CFI
+from .conditional_feature_importance import CFI, cfi_analysis
 from .desparsified_lasso import DesparsifiedLasso, desparsified_lasso, reid
 from .distilled_conditional_randomization_test import D0CRT, d0crt
 from .ensemble_clustered_inference import (
@@ -8,8 +8,8 @@
     ensemble_clustered_inference_pvalue,
 )
 from .knockoffs import ModelXKnockoff
-from .leave_one_covariate_out import LOCO
-from .permutation_feature_importance import PFI
+from .leave_one_covariate_out import LOCO, loco_analysis
+from .permutation_feature_importance import PFI, pfi_analysis
 from .statistical_tools.aggregation import quantile_aggregation
 
 try:
@@ -30,6 +30,9 @@
     "reid",
     "ModelXKnockoff",
     "CFI",
+    "cfi_analysis",
     "LOCO",
+    "loco_analysis",
     "PFI",
+    "pfi_analysis",
 ]

src/hidimstat/_utils/utils.py

@@ -1,7 +1,11 @@
 import numbers
+from functools import partial
 
 import numpy as np
 from numpy.random import RandomState
+from scipy.stats import ttest_1samp, wilcoxon
+
+from hidimstat.statistical_tools.nadeau_bengio_ttest import nadeau_bengio_ttest
 
 
 def _check_vim_predict_method(method):
@@ -33,6 +37,37 @@ def _check_vim_predict_method(method):
         )
 
 
+def get_fitted_attributes(cls):
+    """
+    Get all attributes from a class that end with a single underscore
+    and doesn't start with one underscore.
+
+    Parameters
+    ----------
+    cls : class
+        The class to inspect for attributes.
+
+    Returns
+    -------
+    list
+        A list of attribute names that end with a single underscore but not double underscore.
+    """
+    # Get all attributes and methods of the class
+    all_attributes = dir(cls)
+
+    # Filter out attributes that start with an underscore
+    filtered_attributes = [attr for attr in all_attributes if not attr.startswith("_")]
+
+    # Filter out attributes that do not end with a single underscore
+    result = [
+        attr
+        for attr in filtered_attributes
+        if attr.endswith("_") and not attr.endswith("__")
+    ]
+
+    return result
+
+
 def check_random_state(seed):
     """
     Modified version of sklearn's check_random_state using np.random.Generator.
@@ -105,3 +140,56 @@ def seed_estimator(estimator, random_state=None):
                 setattr(value, "random_state", RandomState(rng.bit_generator))
 
     return estimator
+
+
+def check_statistical_test(statistical_test, test_frac=None):
+    """
+    Validates and returns a test statistic function.
+
+    Parameters
+    ----------
+    statisticcal_test : str or callable
+        If str, must be either 'ttest' or 'wilcoxon'.
+        If callable, must be a function that can be used as a test statistic.
+    test_frac : float, optional
+        The fraction of data used for testing in the Nadeau-Bengio t-test.
+
+    Returns
+    -------
+    callable
+        A function that can be used as a test statistic.
+        For string inputs, returns a partial function of either ttest_1samp or wilcoxon.
+        For callable inputs, returns the input function.
+
+    Raises
+    ------
+    ValueError
+        If test is a string but not one of the supported test names ('ttest' or 'wilcoxon').
+    ValueError
+        If test is neither a string nor a callable.
+    """
+    if isinstance(statistical_test, str):
+        if statistical_test == "ttest":
+            return partial(ttest_1samp, popmean=0, alternative="greater", axis=1)
+        elif statistical_test == "wilcoxon":
+            return partial(wilcoxon, alternative="greater", axis=1)
+        elif statistical_test == "nb-ttest":
+            return partial(
+                nadeau_bengio_ttest,
+                popmean=0,
+                test_frac=test_frac,
+                alternative="greater",
+                axis=1,
+            )
+        else:
+            raise ValueError(f"the test '{statistical_test}' is not supported")
+    elif callable(statistical_test):
+        return statistical_test
+    else:
+        raise ValueError(
+            f"Unsupported value for 'statistical_test'."
+            f"The provided argument was '{statistical_test}'. "
+            f"Please choose from the following valid options: "
+            f"string values ('ttest', 'wilcoxon', 'nb-ttest') "
+            f"or a custom callable function with a `scipy.stats` API-compatible signature."
+        )