ENH & MTN

 - Use BinaryClassificationRisk to compute risk
 - Use warning instead of error when risk is not controled. Throw error when predicting
 - Remove useless check on lambda=None in ltt_procedure
 - Remove useless p_values from ltt_procedure outputs
 - Add possibility to pass an array of n_obs to ltt_procedure and subsequent p-values calculations (needed for binary classification)

Author: Valentin-Laurent

mapie/__init__.py

@@ -4,7 +4,6 @@
     regression,
     utils,
     risk_control,
-    risk_control_draft,
     calibration,
     subsample,
 )
@@ -14,7 +13,6 @@
     "regression",
     "classification",
     "risk_control",
-    "risk_control_draft",
     "calibration",
     "metrics",
     "utils",

mapie/control_risk/ltt.py

@@ -1,5 +1,5 @@
 import warnings
-from typing import Any, List, Optional, Tuple
+from typing import Any, List, Optional, Tuple, Union
 
 import numpy as np
 
@@ -9,29 +9,26 @@
 
 
 def ltt_procedure(
-    r_hat: NDArray[np.float32],
-    alpha_np: NDArray[np.float32],
-    delta: Optional[float],
-    n_obs: int,
-    binary: bool = False,  # TODO: maybe should pass p_values fonction instead
-) -> Tuple[List[List[Any]], NDArray[np.float32]]:
+    r_hat: NDArray[float],
+    alpha_np: NDArray[float],
+    delta: float,
+    n_obs: Union[int, NDArray],
+    binary: bool = False,
+) -> List[List[Any]]:
     """
     Apply the Learn-Then-Test procedure for risk control.
     Note that we will do a multiple test for ``r_hat`` that are
     less than level ``alpha_np``.
     The procedure follows the instructions in [1]:
-        - Calculate p-values for each lambdas descretized
-        - Apply a family wise error rate algorithm,
-        here Bonferonni correction
-        - Return the index lambdas that give you the control
-        at alpha level
+        - Calculate p-values for each lambdas discretized
+        - Apply a family wise error rate algorithm, here Bonferonni correction
+        - Return the index lambdas that give you the control at alpha level
 
     Parameters
     ----------
     r_hat: NDArray of shape (n_lambdas, ).
-        Empirical risk with respect
-        to the lambdas.
-        Here lambdas are thresholds that impact decision making,
+        Empirical risk with respect to the lambdas.
+        Here lambdas are thresholds that impact decision-making,
         therefore empirical risk.
 
     alpha_np: NDArray of shape (n_alpha, ).
@@ -44,34 +41,34 @@ def ltt_procedure(
         Correspond to proportion of failure we don't
         want to exceed.
 
+    n_obs: Union[int, NDArray]
+        Correspond to the number of observations used to compute the risk.
+        In the case of a conditional loss, n_obs must be the
+        number of effective observations used to compute the empirical risk
+        for each lambda, hence of shape (n_lambdas, ).
+
+    binary: bool, default=False
+        Must be True if the loss associated to the risk is binary.
+
     Returns
     -------
     valid_index: List[List[Any]].
-        Contain the valid index that satisfy fwer control
+        Contain the valid index that satisfy FWER control
         for each alpha (length aren't the same for each alpha).
 
-    p_values: NDArray of shape (n_lambda, n_alpha).
-        Contains the values of p_value for different alpha.
-
     References
     ----------
     [1] Angelopoulos, A. N., Bates, S., Candès, E. J., Jordan,
     M. I., & Lei, L. (2021). Learn then test:
     "Calibrating predictive algorithms to achieve risk control".
     """
-    if delta is None:
-        raise ValueError(
-            "Invalid delta: delta cannot be None while"
-            + " controlling precision with LTT. "
-        )
     p_values = compute_hoeffdding_bentkus_p_value(r_hat, n_obs, alpha_np, binary)
     N = len(p_values)
     valid_index = []
     for i in range(len(alpha_np)):
         l_index = np.where(p_values[:, i] <= delta/N)[0].tolist()
         valid_index.append(l_index)
-    return valid_index, p_values  # TODO : p_values is not used, we could remove it
-    # Or return corrected p_values
+    return valid_index
 
 
 def find_lambda_control_star(

mapie/control_risk/p_values.py

@@ -8,11 +8,11 @@
 
 
 def compute_hoeffdding_bentkus_p_value(
-    r_hat: NDArray[np.float32],
-    n_obs: int,
-    alpha: Union[float, NDArray[np.float32]],
+    r_hat: NDArray[float],
+    n_obs: Union[int, NDArray],
+    alpha: Union[float, NDArray[float]],
     binary: bool = False,
-) -> NDArray[np.float32]:
+) -> NDArray[float]:
     """
     The method computes the p_values according to
     the Hoeffding_Bentkus inequality for each
@@ -30,16 +30,23 @@ def compute_hoeffdding_bentkus_p_value(
         Here lambdas are thresholds that impact decision
         making and therefore empirical risk.
 
-    n_obs: int.
-        Correspond to the number of observations in
-        dataset.
+    n_obs: Union[int, NDArray]
+        Correspond to the number of observations used to compute the risk.
+        In the case of a conditional loss, n_obs must be the
+        number of effective observations used to compute the empirical risk
+        for each lambda, hence of shape (n_lambdas, ).
 
     alpha: Union[float, Iterable[float]].
         Contains the different alphas control level.
         The empirical risk must be less than alpha.
         If it is a iterable, it is a NDArray of shape
         (n_alpha, ).
 
+    binary: bool, default=False
+        Must be True if the loss associated to the risk is binary.
+        If True, we use a tighter version of the Bentkus p-value, valid when the
+        loss associated to the risk is binary. See section 3.2 of [1].
+
     Returns
     -------
     hb_p_values: NDArray of shape (n_lambda, n_alpha).
@@ -62,9 +69,17 @@ def compute_hoeffdding_bentkus_p_value(
         len(r_hat),
         axis=0
     )
+    if isinstance(n_obs, int):
+        n_obs = np.full_like(r_hat, n_obs, dtype=float)
+    n_obs_repeat = np.repeat(
+        np.expand_dims(n_obs, axis=1),
+        len(alpha_np),
+        axis=1
+    )
+
     hoeffding_p_value = np.exp(
-        -n_obs * _h1(
-            np.where(  # TODO : shouldn't we use np.minimum ?
+        -n_obs_repeat * _h1(
+            np.where(
                 r_hat_repeat > alpha_repeat,
                 alpha_repeat,
                 r_hat_repeat
@@ -74,9 +89,9 @@ def compute_hoeffdding_bentkus_p_value(
     )
     factor = 1 if binary else np.e
     bentkus_p_value = factor * binom.cdf(
-        np.ceil(n_obs * r_hat_repeat), n_obs, alpha_repeat
+        np.ceil(n_obs_repeat * r_hat_repeat), n_obs, alpha_repeat
     )
-    hb_p_value = np.where(  # TODO : shouldn't we use np.minimum ?
+    hb_p_value = np.where(
         bentkus_p_value > hoeffding_p_value,
         hoeffding_p_value,
         bentkus_p_value
@@ -85,8 +100,8 @@ def compute_hoeffdding_bentkus_p_value(
 
 
 def _h1(
-    r_hats: NDArray[np.float32], alphas: NDArray[np.float32]
-) -> NDArray[np.float32]:
+    r_hats: NDArray[float], alphas: NDArray[float]
+) -> NDArray[float]:
     """
     This function allow us to compute
     the tighter version of hoeffding inequality.
@@ -113,7 +128,7 @@ def _h1(
 
     Returns
     -------
-    NDArray of shape a(n_lambdas, n_alpha).
+    NDArray of shape (n_lambdas, n_alpha).
     """
     elt1 = np.zeros_like(r_hats, dtype=float)
 

mapie/risk_control.py

@@ -681,8 +681,8 @@ def predict(
         if self.metric_control == 'precision':
             self.n_obs = len(self.risks)
             self.r_hat = self.risks.mean(axis=0)
-            self.valid_index, self.p_values = ltt_procedure(
-                self.r_hat, alpha_np, delta, self.n_obs
+            self.valid_index = ltt_procedure(
+                self.r_hat, alpha_np, cast(float, delta), self.n_obs
             )
             self._check_valid_index(alpha_np)
             self.lambdas_star, self.r_star = find_lambda_control_star(
@@ -724,8 +724,8 @@ def __init__(
 
     def get_value_and_effective_sample_size(
         self,
-        y_true: NDArray[int], # shape (n_samples,), values in {0, 1}
-        y_pred: NDArray[int], # shape (n_samples,), values in {0, 1}
+        y_true: NDArray[int],  # shape (n_samples,), values in {0, 1}
+        y_pred: NDArray[int],  # shape (n_samples,), values in {0, 1}
     ) -> Optional[Tuple[float, int]]:
         # float between 0 and 1, int between 0 and len(y_true)
         risk_occurrences = [
@@ -765,4 +765,10 @@ def get_value_and_effective_sample_size(
     risk_occurrence=lambda y_true, y_pred: int(y_pred == y_true),
     risk_condition=lambda y_true, y_pred: y_true == 1,
     higher_is_better=True,
-)
+)
+
+_automatic_best_predict_param_choice = {
+    precision: recall,
+    recall: precision,
+    accuracy: accuracy,
+}