Update classification_coverage_score (#686)

This PR implements the transition from classification_coverage_score to classification_coverage_score_v2:

* Replace all instances of classification_coverage_score with classification_coverage_score_v2 in the codebase and associated tests;

* Remove the original classification_coverage_score function;

* Rename classification_coverage_score_v2 to classification_coverage_score;

* Update migration guide about this change.

Author: FaustinPulveric

doc/api.rst

@@ -71,7 +71,6 @@ Classification Metrics
    :template: function.rst
 
    mapie.metrics.classification.classification_coverage_score
-   mapie.metrics.classification.classification_coverage_score_v2
    mapie.metrics.classification.classification_mean_width_score
    mapie.metrics.classification.classification_ssc
    mapie.metrics.classification.classification_ssc_score

doc/quick_start.rst

@@ -138,9 +138,9 @@ Similarly, it's possible to do the same for a basic classification problem.
 
 .. code:: python
 
-    from mapie.metrics import classification_coverage_score_v2
+    from mapie.metrics import classification_coverage_score
 
-    coverage_scores = classification_coverage_score_v2(y_test, y_pis)
+    coverage_scores = classification_coverage_score(y_test, y_pis)
 
 .. code:: python
 

doc/v1_migration_guide.rst

@@ -209,6 +209,12 @@ Additionally, a number of classification and regression functions have been upda
 - **v0.x**: Took the prediction sets in an array of shape (n_samples, n_class) for a given confidence level as input, and returned the effective mean width as a float.
 - **v1**: Now takes the prediction sets in an array of shape (n_samples, n_class, n_confidence_level) as input, and returns the effective mean width for each confidence level as an array of shape (n_confidence_level,).
 
+``classification_coverage_score``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- **v0.x**: Had two separate versions: ``classification_coverage_score`` and ``classification_coverage_v2``.
+- **v1**: ``classification_coverage_score`` now corresponds to MAPIE v0.x's ``classification_coverage_score_v2``.
+
 ``regression_mean_width``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

examples/classification/1-quickstart/plot_comp_methods_on_2d_dataset.py

@@ -56,7 +56,7 @@
 from numpy.typing import NDArray
 from mapie.classification import SplitConformalClassifier
 from mapie.metrics.classification import (
-    classification_coverage_score_v2,
+    classification_coverage_score,
     classification_mean_width_score,
 )
 
@@ -264,7 +264,7 @@ def plot_results(
         X, conformity_score_params={"include_last_label": "randomized"}
     )
     coverage[conformity_score] = [
-        classification_coverage_score_v2(y, y_ps_mapie[conformity_score])
+        classification_coverage_score(y, y_ps_mapie[conformity_score])
     ]
     mean_width[conformity_score] = classification_mean_width_score(
         y_ps_mapie[conformity_score]

examples/classification/4-tutorials/plot_crossconformal.py

@@ -38,7 +38,7 @@
 from numpy.typing import NDArray
 from mapie.classification import SplitConformalClassifier, CrossConformalClassifier
 from mapie.metrics.classification import (
-    classification_coverage_score_v2,
+    classification_coverage_score,
     classification_mean_width_score,
 )
 
@@ -193,7 +193,7 @@ def plot_results(
             vmin=0,
             vmax=3
         )
-        coverage = classification_coverage_score_v2(
+        coverage = classification_coverage_score(
             y_test2, mapie.predict_set(X_test2)[1][:, :, 89]
         )[0]
         axs[i].set_title(f"coverage = {coverage:.3f}")
@@ -266,7 +266,7 @@ def plot_coverage_width(
 split_coverages = np.array(
     [
         [
-            classification_coverage_score_v2(y_test_distrib, y_ps)
+            classification_coverage_score(y_test_distrib, y_ps)
             for _, y_ps in y_ps2.items()
         ] for _, y_ps2 in y_ps_mapies.items()
     ]
@@ -414,7 +414,7 @@ def plot_coverage_width(
 
 for strategy, y_ps_ in y_ps.items():
     coverages[strategy] = np.array(
-        classification_coverage_score_v2(y_test_distrib, y_ps_)
+        classification_coverage_score(y_test_distrib, y_ps_)
     )
     widths[strategy] = np.array(
         classification_mean_width_score(y_ps_)

examples/classification/4-tutorials/plot_main-tutorial-binary-classification.py

@@ -37,7 +37,7 @@
 from mapie.classification import SplitConformalClassifier
 from mapie.utils import train_conformalize_test_split
 from mapie.metrics.classification import (
-    classification_coverage_score_v2,
+    classification_coverage_score,
     classification_mean_width_score,
 )
 
@@ -304,7 +304,7 @@ def plot_results(
 mapie_clf.conformalize(X_c2, y_c2)
 _, y_ps_mapie = mapie_clf.predict_set(X)
 
-coverage = classification_coverage_score_v2(y, y_ps_mapie)
+coverage = classification_coverage_score(y, y_ps_mapie)
 mean_width = classification_mean_width_score(y_ps_mapie)
 
 

examples/classification/4-tutorials/plot_main-tutorial-classification.py

@@ -24,7 +24,7 @@
 from mapie.classification import SplitConformalClassifier
 from mapie.utils import train_conformalize_test_split
 from mapie.metrics.classification import (
-    classification_coverage_score_v2,
+    classification_coverage_score,
     classification_mean_width_score,
 )
 
@@ -222,7 +222,7 @@ def plot_results(confidence_levels, X, y_pred, y_ps):
 )
 mapie_score2.conformalize(X_conf, y_conf)
 _, y_ps_score2 = mapie_score2.predict_set(X_test)
-coverages_score = classification_coverage_score_v2(y_test, y_ps_score2)
+coverages_score = classification_coverage_score(y_test, y_ps_score2)
 widths_score = classification_mean_width_score(y_ps_score2)
 
 
@@ -290,7 +290,7 @@ def plot_coverages_widths(confidence_level, coverage, width, conformity_score):
 _, y_ps_aps2 = mapie_aps2.predict_set(
     X_test, conformity_score_params={"include_last_label": "randomized"}
 )
-coverages_aps = classification_coverage_score_v2(y_test, y_ps_aps2)
+coverages_aps = classification_coverage_score(y_test, y_ps_aps2)
 widths_aps = classification_mean_width_score(y_ps_aps2)
 
 plot_coverages_widths(

mapie/metrics/classification.py

@@ -2,7 +2,7 @@
 
 import numpy as np
 from numpy.typing import ArrayLike, NDArray
-from sklearn.utils import column_or_1d, check_array
+from sklearn.utils import column_or_1d
 
 from mapie.utils import (
     _check_arrays_length,
@@ -12,63 +12,6 @@
 )
 
 
-def classification_coverage_score(
-    y_true: ArrayLike,
-    y_pred_set: ArrayLike
-) -> float:
-    """
-    Effective coverage score obtained by the prediction sets.
-
-    The effective coverage is obtained by estimating the fraction
-    of true labels that lie within the prediction sets.
-
-    Parameters
-    ----------
-    y_true: ArrayLike of shape (n_samples,)
-        True labels.
-    y_pred_set: ArrayLike of shape (n_samples, n_class)
-        Prediction sets given by booleans of labels.
-
-    Returns
-    -------
-    float
-        Effective coverage obtained by the prediction sets.
-
-    Examples
-    --------
-    >>> from mapie.metrics.classification import classification_coverage_score
-    >>> import numpy as np
-    >>> y_true = np.array([3, 3, 1, 2, 2])
-    >>> y_pred_set = np.array([
-    ...     [False, False,  True,  True],
-    ...     [False,  True, False,  True],
-    ...     [False,  True,  True, False],
-    ...     [False, False,  True,  True],
-    ...     [False,  True, False,  True]
-    ... ])
-    >>> print(classification_coverage_score(y_true, y_pred_set))
-    0.8
-    """
-    y_true = cast(NDArray, column_or_1d(y_true))
-    y_pred_set = cast(
-        NDArray,
-        check_array(
-            y_pred_set, dtype=["bool"]
-        )
-    )
-
-    _check_arrays_length(y_true, y_pred_set)
-    _check_array_nan(y_true)
-    _check_array_inf(y_true)
-    _check_array_nan(y_pred_set)
-    _check_array_inf(y_pred_set)
-
-    coverage = np.take_along_axis(
-        y_pred_set, y_true.reshape(-1, 1), axis=1
-    ).mean()
-    return float(coverage)
-
-
 def classification_mean_width_score(y_pred_set: ArrayLike) -> float:
     """
     Mean width of prediction set output by
@@ -106,7 +49,7 @@ def classification_mean_width_score(y_pred_set: ArrayLike) -> float:
     return mean_width
 
 
-def classification_coverage_score_v2(
+def classification_coverage_score(
     y_true: NDArray,
     y_pred_set: NDArray
 ) -> NDArray:
@@ -141,7 +84,7 @@ def classification_coverage_score_v2(
 
     Examples
     --------
-    >>> from mapie.metrics.classification import classification_coverage_score_v2
+    >>> from mapie.metrics.classification import classification_coverage_score
     >>> from mapie.classification import SplitConformalClassifier
     >>> from mapie.utils import train_conformalize_test_split
     >>> from sklearn.datasets import make_classification
@@ -163,7 +106,7 @@ def classification_coverage_score_v2(
     ... ).fit(X_train, y_train).conformalize(X_conformalize, y_conformalize)
 
     >>> predicted_points, predicted_sets = mapie_classifier.predict_set(X_test)
-    >>> coverage = classification_coverage_score_v2(y_test, predicted_sets)[0]
+    >>> coverage = classification_coverage_score(y_test, predicted_sets)[0]
     """
     _check_arrays_length(y_true, y_pred_set)
     _check_array_nan(y_true)
@@ -258,7 +201,7 @@ def classification_ssc(
         ]
 
         for i, indexes in enumerate(indexes_bybins):
-            coverages[alpha, i] = classification_coverage_score_v2(
+            coverages[alpha, i] = classification_coverage_score(
                 y_true[indexes],
                 np.take_along_axis(
                     y_pred_set[:, :, alpha],

mapie/tests/test_classification.py

@@ -1460,7 +1460,7 @@ def test_toy_dataset_predictions(strategy: str) -> None:
     )
     np.testing.assert_allclose(y_ps[:, :, 0], y_toy_mapie[strategy])
     np.testing.assert_allclose(
-        classification_coverage_score(y_toy, y_ps[:, :, 0]),
+        classification_coverage_score(y_toy, y_ps)[0],
         COVERAGES[strategy],
     )
 
@@ -1482,7 +1482,7 @@ def test_large_dataset_predictions(strategy: str) -> None:
         agg_scores=args_predict["agg_scores"]
     )
     np.testing.assert_allclose(
-        classification_coverage_score(y, y_ps[:, :, 0]),
+        classification_coverage_score(y, y_ps)[0],
         LARGE_COVERAGES[strategy], rtol=1e-2
     )
 
@@ -1507,7 +1507,7 @@ def test_toy_binary_dataset_predictions(strategy: str) -> None:
     )
     np.testing.assert_allclose(y_ps[:, :, 0], y_toy_binary_mapie[strategy])
     np.testing.assert_allclose(
-        classification_coverage_score(y_toy_binary, y_ps[:, :, 0]),
+        classification_coverage_score(y_toy_binary, y_ps)[0],
         COVERAGES_BINARY[strategy],
     )
 

mapie/tests/test_metrics.py

@@ -26,9 +26,8 @@
     top_label_ece,
 )
 from mapie.metrics.classification import (
-    classification_coverage_score,
     classification_mean_width_score,
-    classification_coverage_score_v2,
+    classification_coverage_score,
     classification_ssc, classification_ssc_score,
 )
 from mapie.metrics.regression import (
@@ -265,10 +264,6 @@ def test_regression_toydata_coverage_score() -> None:
 
 def test_classification_y_true_shape() -> None:
     """Test shape of y_true."""
-    with pytest.raises(ValueError, match=r".*y should be a 1d array*"):
-        classification_coverage_score(
-            np.tile(y_true_class, (2, 1)), y_pred_set
-        )
     with pytest.raises(ValueError, match=r".*y should be a 1d array*"):
         classification_ssc(np.tile(y_true_class, (2, 1)), y_pred_set_2alphas)
     with pytest.raises(ValueError, match=r".*are arrays with different len*"):
@@ -278,8 +273,6 @@ def test_classification_y_true_shape() -> None:
 
 def test_classification_y_pred_set_shape() -> None:
     """Test shape of y_pred_set."""
-    with pytest.raises(ValueError, match=r".*Expected 2D array*"):
-        classification_coverage_score(y_true_class, y_pred_set[:, 0])
     with pytest.raises(ValueError, match=r".*should be a 3D array*"):
         classification_ssc(y_true_class, y_pred_set[:, 0])
     with pytest.raises(ValueError, match=r".*should be a 3D array*"):
@@ -305,19 +298,7 @@ def test_classification_valid_input_shape() -> None:
 
 def test_classification_toydata() -> None:
     """Test coverage_score for toy data."""
-    assert classification_coverage_score(y_true_class, y_pred_set) == 0.8
-
-
-def test_classification_ytrue_type() -> None:
-    """Test that list(y_true_class) gives right coverage."""
-    scr = classification_coverage_score(list(y_true_class), y_pred_set)
-    assert scr == 0.8
-
-
-def test_classification_y_pred_set_type() -> None:
-    """Test that list(y_pred_set) gives right coverage."""
-    scr = classification_coverage_score(y_true_class, list(y_pred_set))
-    assert scr == 0.8
+    assert classification_coverage_score(y_true_class, y_pred_set)[0] == 0.8
 
 
 def test_classification_mean_width_score_toydata() -> None:
@@ -550,28 +531,15 @@ def test_regression_coverage_score_intervals_invalid_shape() -> None:
         )
 
 
-def test_classification_coverage_v1andv2() -> None:
-    """
-    Test that ``classification_coverage_score`` and
-    ```classification_coverage_score_v2``` returns the same results
-    """
-    cov_v1 = classification_coverage_score(y_true_class, y_pred_set)
-    cov_v2 = classification_coverage_score_v2(
-        np.expand_dims(y_true_class, axis=1),
-        np.expand_dims(y_pred_set, axis=2)
-    )
-    np.testing.assert_allclose(cov_v1, cov_v2[0])
-
-
-def test_classification_coverage_score_v2_ytrue_valid_shape() -> None:
+def test_classification_coverage_score_ytrue_valid_shape() -> None:
     """Test that no error is raised if y_true has a shape (n_samples,)."""
-    classification_coverage_score_v2(y_true_class, y_pred_set_2alphas)
+    classification_coverage_score(y_true_class, y_pred_set_2alphas)
 
 
-def test_classification_coverage_score_v2_ypredset_invalid_shape() -> None:
+def test_classification_coverage_score_ypredset_invalid_shape() -> None:
     """Test that an error is raised if y_pred_set has not the good shape."""
     with pytest.raises(ValueError):
-        classification_coverage_score_v2(
+        classification_coverage_score(
             np.expand_dims(y_true_class, axis=1), y_pred_set[:, 0]
         )