[API 2]: CFI, PFI, LOCO

docs/src/api.rst

@@ -52,8 +52,8 @@ Samplers
    :toctree: ./generated/api/class/
    :template: class.rst
 
-   ~statistical_tools.ConditionalSampler
-   ~statistical_tools.GaussianKnockoffs
+   ~samplers.ConditionalSampler
+   ~samplers.GaussianKnockoffs
 
 Helper Functions
 ================

examples/plot_conditional_vs_marginal_xor_data.py

@@ -131,7 +131,7 @@
         random_state=0,
     )
     vim.fit(X_train, y_train)
-    importances.append(vim.importance(X_test, y_test)["importance"])
+    importances.append(vim.importance(X_test, y_test))
 
 importances = np.array(importances).T
 

examples/plot_diabetes_variable_importance_example.py

@@ -63,7 +63,7 @@
 import numpy as np
 from sklearn.base import clone
 from sklearn.linear_model import LogisticRegressionCV, RidgeCV
-from sklearn.metrics import r2_score, root_mean_squared_error
+from sklearn.metrics import mean_squared_error, r2_score
 from sklearn.model_selection import KFold
 
 n_folds = 5
@@ -78,7 +78,7 @@
     score = r2_score(
         y_true=y[test_index], y_pred=regressor_list[i].predict(X[test_index])
     )
-    mse = root_mean_squared_error(
+    mse = mean_squared_error(
         y_true=y[test_index], y_pred=regressor_list[i].predict(X[test_index])
     )
 
@@ -166,14 +166,14 @@
 import pandas as pd
 from scipy.stats import ttest_1samp
 
-cfi_vim_arr = np.array([x["importance"] for x in cfi_importance_list]) / 2
+cfi_vim_arr = np.array(cfi_importance_list) / 2
 cfi_pval = ttest_1samp(cfi_vim_arr, 0, alternative="greater").pvalue
 
 vim = [
     pd.DataFrame(
         {
             "var": np.arange(cfi_vim_arr.shape[1]),
-            "importance": x["importance"],
+            "importance": x,
             "fold": i,
             "pval": cfi_pval,
             "method": "CFI",
@@ -182,14 +182,14 @@
     for x in cfi_importance_list
 ]
 
-loco_vim_arr = np.array([x["importance"] for x in loco_importance_list])
+loco_vim_arr = np.array(loco_importance_list)
 loco_pval = ttest_1samp(loco_vim_arr, 0, alternative="greater").pvalue
 
 vim += [
     pd.DataFrame(
         {
             "var": np.arange(loco_vim_arr.shape[1]),
-            "importance": x["importance"],
+            "importance": x,
             "fold": i,
             "pval": loco_pval,
             "method": "LOCO",
@@ -198,14 +198,14 @@
     for x in loco_importance_list
 ]
 
-pfi_vim_arr = np.array([x["importance"] for x in pfi_importance_list])
+pfi_vim_arr = np.array(pfi_importance_list)
 pfi_pval = ttest_1samp(pfi_vim_arr, 0, alternative="greater").pvalue
 
 vim += [
     pd.DataFrame(
         {
             "var": np.arange(pfi_vim_arr.shape[1]),
-            "importance": x["importance"],
+            "importance": x,
             "fold": i,
             "pval": pfi_pval,
             "method": "PFI",

examples/plot_importance_classification_iris.py

@@ -107,7 +107,7 @@ def run_one_fold(
         )
 
     vim.fit(X[train_index], y[train_index])
-    importance = vim.importance(X[test_index], y[test_index])["importance"]
+    importance = vim.importance(X[test_index], y[test_index])
 
     return pd.DataFrame(
         {

examples/plot_loco.py

@@ -82,7 +82,7 @@
     # importance. This process is repeated for all features to assess their individual
     # contributions.
     loco.fit(X_train, y_train)
-    importances = loco.importance(X_test, y_test)["importance"]
+    importances = loco.importance(X_test, y_test)
     df_list.append(
         pd.DataFrame(
             {

examples/plot_model_agnostic_importance.py

@@ -122,12 +122,8 @@
     vim_linear.fit(X[train], y[train])
     vim_non_linear.fit(X[train], y[train])
 
-    importances_linear.append(
-        vim_linear.importance(X[test], y[test])["importance"],
-    )
-    importances_non_linear.append(
-        vim_non_linear.importance(X[test], y[test])["importance"]
-    )
+    importances_linear.append(vim_linear.importance(X[test], y[test]))
+    importances_non_linear.append(vim_non_linear.importance(X[test], y[test]))
 
 
 # %%

examples/plot_pitfalls_permutation_importance.py

@@ -144,7 +144,7 @@
     )
     pfi.fit(X_test, y_test)
 
-    permutation_importances.append(pfi.importance(X_test, y_test)["importance"])
+    permutation_importances.append(pfi.importance(X_test, y_test))
 permutation_importances = np.stack(permutation_importances)
 pval_pfi = ttest_1samp(
     permutation_importances, 0.0, axis=0, alternative="greater"
@@ -216,7 +216,7 @@
     )
     cfi.fit(X_test, y_test)
 
-    conditional_importances.append(cfi.importance(X_test, y_test)["importance"])
+    conditional_importances.append(cfi.importance(X_test, y_test))
 
 
 cfi_pval = ttest_1samp(
@@ -267,7 +267,7 @@
 
 from matplotlib.lines import Line2D
 
-from hidimstat.statistical_tools.conditional_sampling import ConditionalSampler
+from hidimstat.samplers.conditional_sampling import ConditionalSampler
 
 X_train, X_test = train_test_split(
     X,

src/hidimstat/__init__.py

@@ -1,4 +1,4 @@
-from .conditional_feature_importance import CFI
+from .conditional_feature_importance import CFI, cfi
 from .desparsified_lasso import (
     desparsified_group_lasso_pvalue,
     desparsified_lasso,
@@ -17,9 +17,9 @@
     model_x_knockoff_bootstrap_quantile,
     model_x_knockoff_pvalue,
 )
-from .leave_one_covariate_out import LOCO
+from .leave_one_covariate_out import LOCO, loco
 from .noise_std import reid
-from .permutation_feature_importance import PFI
+from .permutation_feature_importance import PFI, pfi
 from .statistical_tools.aggregation import quantile_aggregation
 
 try:
@@ -44,6 +44,9 @@
     "model_x_knockoff_bootstrap_quantile",
     "model_x_knockoff_bootstrap_e_value",
     "CFI",
+    "cfi",
     "LOCO",
+    "loco",
     "PFI",
+    "pfi",
 ]

src/hidimstat/_utils/utils.py

@@ -1,7 +1,11 @@
 import numbers
+from functools import partial
 
 import numpy as np
 from numpy.random import RandomState
+from scipy.stats import ttest_1samp, wilcoxon
+
+from hidimstat.statistical_tools.nadeau_bengio_ttest import nadeau_bengio_ttest
 
 
 def _check_vim_predict_method(method):
@@ -33,6 +37,37 @@ def _check_vim_predict_method(method):
         )
 
 
+def get_fitted_attributes(cls):
+    """
+    Get all attributes from a class that end with a single underscore
+    and doesn't start with one underscore.
+
+    Parameters
+    ----------
+    cls : class
+        The class to inspect for attributes.
+
+    Returns
+    -------
+    list
+        A list of attribute names that end with a single underscore but not double underscore.
+    """
+    # Get all attributes and methods of the class
+    all_attributes = dir(cls)
+
+    # Filter out attributes that start with an underscore
+    filtered_attributes = [attr for attr in all_attributes if not attr.startswith("_")]
+
+    # Filter out attributes that do not end with a single underscore
+    result = [
+        attr
+        for attr in filtered_attributes
+        if attr.endswith("_") and not attr.endswith("__")
+    ]
+
+    return result
+
+
 def check_random_state(seed):
     """
     Modified version of sklearn's check_random_state using np.random.Generator.
@@ -105,3 +140,54 @@ def seed_estimator(estimator, random_state=None):
                 setattr(value, "random_state", RandomState(rng.bit_generator))
 
     return estimator
+
+
+def check_statistical_test(statistical_test):
+    """
+    Validates and returns a test statistic function.
+
+    Parameters
+    ----------
+    test : str or callable
+        If str, must be either 'ttest' or 'wilcoxon'.
+        If callable, must be a function that can be used as a test statistic.
+
+    Returns
+    -------
+    callable
+        A function that can be used as a test statistic.
+        For string inputs, returns a partial function of either ttest_1samp or wilcoxon.
+        For callable inputs, returns the input function.
+
+    Raises
+    ------
+    ValueError
+        If test is a string but not one of the supported test names ('ttest' or 'wilcoxon').
+    ValueError
+        If test is neither a string nor a callable.
+    """
+    if isinstance(statistical_test, str):
+        if statistical_test == "ttest":
+            return partial(ttest_1samp, popmean=0, alternative="greater", axis=1)
+        elif statistical_test == "wilcoxon":
+            return partial(wilcoxon, alternative="greater", axis=1)
+        elif statistical_test == "nb-ttest":
+            return partial(
+                nadeau_bengio_ttest,
+                popmean=0,
+                test_frac=0.1 / 0.9,
+                alternative="greater",
+                axis=1,
+            )
+        else:
+            raise ValueError(f"the test '{statistical_test}' is not supported")
+    elif callable(statistical_test):
+        return statistical_test
+    else:
+        raise ValueError(
+            f"Unsupported value for 'statistical_test'."
+            f"The provided argument was '{statistical_test}'. "
+            f"Please choose from the following valid options: "
+            f"string values ('ttest', 'wilcoxon', 'nb-ttest') "
+            f"or a custom callable function with a `scipy.stats` API-compatible signature."
+        )