MTN: Remove deprecated parameter "method" of _MapieClassifier (#669)

Author: Valentin-Laurent

examples/classification/3-scientific-articles/plot_sadinle2019_example.py

@@ -26,6 +26,7 @@
 from sklearn.naive_bayes import GaussianNB
 
 from mapie.classification import _MapieClassifier
+from mapie.conformity_scores import LACConformityScore
 
 # Create training set from multivariate normal distribution
 centers = [(0, 3.5), (-2, 0), (2, 0)]
@@ -56,7 +57,11 @@
 y_pred = clf.predict(X_test)
 y_pred_proba = clf.predict_proba(X_test)
 y_pred_proba_max = np.max(y_pred_proba, axis=1)
-mapie = _MapieClassifier(estimator=clf, cv="prefit", method="lac")
+mapie = _MapieClassifier(
+    estimator=clf,
+    cv="prefit",
+    conformity_score=LACConformityScore()
+)
 mapie.fit(X_train, y_train)
 y_pred_mapie, y_ps_mapie = mapie.predict(X_test, alpha=alpha)
 

examples/classification/4-tutorials/plot_crossconformal.py

@@ -37,6 +37,7 @@
 
 from numpy.typing import NDArray
 from mapie.classification import _MapieClassifier
+from mapie.conformity_scores import LACConformityScore, APSConformityScore
 from mapie.metrics.classification import (
     classification_coverage_score,
     classification_mean_width_score,
@@ -110,21 +111,30 @@
 
 kf = KFold(n_splits=5, shuffle=True)
 clfs, mapies, y_preds, y_ps_mapies = {}, {}, {}, {}
-methods = ["lac", "aps"]
+conformity_scores = [LACConformityScore(), APSConformityScore()]
 alpha = np.arange(0.01, 1, 0.01)
-for method in methods:
+for conformity_score in conformity_scores:
     clfs_, mapies_, y_preds_, y_ps_mapies_ = {}, {}, {}, {}
     for fold, (train_index, calib_index) in enumerate(kf.split(X_train)):
         clf = GaussianNB().fit(X_train[train_index], y_train[train_index])
         clfs_[fold] = clf
-        mapie = _MapieClassifier(estimator=clf, cv="prefit", method=method)
+        mapie = _MapieClassifier(
+            estimator=clf,
+            cv="prefit",
+            conformity_score=conformity_score
+        )
         mapie.fit(X_train[calib_index], y_train[calib_index])
         mapies_[fold] = mapie
         y_pred_mapie, y_ps_mapie = mapie.predict(
             X_test_distrib, alpha=alpha, include_last_label="randomized"
         )
         y_preds_[fold], y_ps_mapies_[fold] = y_pred_mapie, y_ps_mapie
-    clfs[method], mapies[method], y_preds[method], y_ps_mapies[method] = (
+    (
+        clfs[conformity_score],
+        mapies[conformity_score],
+        y_preds[conformity_score],
+        y_ps_mapies[conformity_score]
+    ) = (
         clfs_, mapies_, y_preds_, y_ps_mapies_
     )
 
@@ -134,23 +144,23 @@
 # set and the estimated quantile for ``alpha`` = 0.1.
 
 
-fig, axs = plt.subplots(1, len(mapies["lac"]), figsize=(20, 4))
-for i, (key, mapie) in enumerate(mapies["lac"].items()):
+fig, axs = plt.subplots(1, len(mapies[conformity_scores[0]]), figsize=(20, 4))
+for i, (key, mapie) in enumerate(mapies[conformity_scores[0]].items()):
     quantiles = mapie.conformity_score_function_.quantiles_[9]
     axs[i].set_xlabel("Conformity scores")
     axs[i].hist(mapie.conformity_scores_)
     axs[i].axvline(quantiles, ls="--", color="k")
     axs[i].set_title(f"split={key}\nquantile={quantiles:.3f}")
 plt.suptitle(
     "Distribution of scores on each calibration fold for the "
-    f"{methods[0]} method"
+    f"{conformity_scores[0]} method"
 )
 plt.show()
 
 
 ##############################################################################
 # We notice that the estimated quantile slightly varies among the calibration
-# sets for the two methods explored here, suggesting that the
+# sets for the two conformity scores explored here, suggesting that the
 # train/calibration splitting can slightly impact our results.
 #
 # Let's now visualize this impact on the number of labels included in each
@@ -202,7 +212,7 @@ def plot_results(
 
 
 plot_results(
-    mapies["lac"],
+    mapies[conformity_scores[0]],
     X_test,
     X_test_distrib,
     y_test_distrib,
@@ -211,7 +221,7 @@ def plot_results(
 )
 
 plot_results(
-    mapies["aps"],
+    mapies[conformity_scores[1]],
     X_test,
     X_test_distrib,
     y_test_distrib,
@@ -338,19 +348,19 @@ def plot_coverage_width(
 
 STRATEGIES = {
     "score_cv_mean": (
-        Params(method="lac", cv=kf, random_state=42),
+        Params(conformity_score=LACConformityScore(), cv=kf, random_state=42),
         ParamsPredict(include_last_label=False, agg_scores="mean")
     ),
     "score_cv_crossval": (
-        Params(method="lac", cv=kf, random_state=42),
+        Params(conformity_score=LACConformityScore(), cv=kf, random_state=42),
         ParamsPredict(include_last_label=False, agg_scores="crossval")
     ),
     "cum_score_cv_mean": (
-        Params(method="aps", cv=kf, random_state=42),
+        Params(conformity_score=APSConformityScore(), cv=kf, random_state=42),
         ParamsPredict(include_last_label="randomized", agg_scores="mean")
     ),
     "cum_score_cv_crossval": (
-        Params(method="aps", cv=kf, random_state=42),
+        Params(conformity_score=APSConformityScore(), cv=kf, random_state=42),
         ParamsPredict(include_last_label='randomized', agg_scores="crossval")
     )
 }

examples/classification/4-tutorials/plot_main-tutorial-binary-classification.py

@@ -35,6 +35,7 @@
 
 from numpy.typing import NDArray
 from mapie.classification import _MapieClassifier
+from mapie.conformity_scores import LACConformityScore
 from mapie.metrics.classification import (
     classification_coverage_score,
     classification_mean_width_score,
@@ -122,7 +123,7 @@
 # so that scores can be interpreted as probabilities
 # (see documentation for more information).
 # Then we apply :class:`~mapie.classification._MapieClassifier` in the
-# calibration data with the methods ``score``
+# calibration data with the LAC conformity_score
 # to the estimator indicating that it has already been fitted with
 # `cv="prefit"`.
 # We then estimate the prediction sets with differents alpha values with a
@@ -139,7 +140,7 @@
 calib.fit(X_c1, y_c1)
 
 mapie_clf = _MapieClassifier(
-    estimator=calib, method='lac', cv='prefit', random_state=42
+    estimator=calib, cv='prefit', random_state=42
 )
 mapie_clf.fit(X_c2, y_c2)
 
@@ -156,7 +157,7 @@
 #   base estimator.
 #
 # - ``y_ps_mapie``: the prediction sets estimated by MAPIE using the "lac"
-#   method.
+#   conformity score.
 #
 # Let's now visualize the distribution of the conformity scores with the two
 # methods with the calculated quantiles for the three alpha values.
@@ -261,7 +262,7 @@ def plot_results(
 
 
 ##############################################################################
-# For the "lac" method, when the class coverage is not large enough, the
+# For the "lac" conformity score, when the class coverage is not large enough, the
 # prediction sets can be empty when the model is uncertain at the border
 # between two classes. These null regions disappear for larger class coverages
 # but ambiguous classification regions arise with both classes included in
@@ -293,7 +294,7 @@ def plot_results(
 calib.fit(X_c1, y_c1)
 
 mapie_clf = _MapieClassifier(
-    estimator=calib, method='lac', cv='prefit', random_state=42
+    estimator=calib, conformity_score=LACConformityScore(), cv='prefit', random_state=42
 )
 mapie_clf.fit(X_c2, y_c2)
 _, y_ps_mapie = mapie_clf.predict(
@@ -341,7 +342,7 @@ def plot_coverages_widths(alpha, coverage, width, method):
 calib.fit(X_c1, y_c1)
 
 mapie_clf = _MapieClassifier(
-    estimator=calib, method='lac', cv='prefit', random_state=42
+    estimator=calib, cv='prefit', random_state=42
 )
 mapie_clf.fit(X_c2, y_c2)
 _, y_ps_mapie = mapie_clf.predict(

examples/classification/4-tutorials/plot_main-tutorial-classification.py

@@ -23,6 +23,7 @@
 from sklearn.naive_bayes import GaussianNB
 
 from mapie.classification import _MapieClassifier
+from mapie.conformity_scores import APSConformityScore
 from mapie.metrics.classification import (
     classification_coverage_score,
     classification_mean_width_score,
@@ -104,7 +105,7 @@
 
 ##############################################################################
 # We fit our training data with a Gaussian Naive Base estimator. And then we
-# apply MAPIE in the calibration data with the method ``score`` to the
+# apply MAPIE in the calibration data with the LAC conformity score to the
 # estimator indicating that it has already been fitted with `cv="prefit"`.
 # We then estimate the prediction sets with differents alpha values with a
 # ``fit`` and ``predict`` process.
@@ -113,7 +114,7 @@
 y_pred = clf.predict(X_test)
 y_pred_proba = clf.predict_proba(X_test)
 y_pred_proba_max = np.max(y_pred_proba, axis=1)
-mapie_score = _MapieClassifier(estimator=clf, cv="prefit", method="lac")
+mapie_score = _MapieClassifier(estimator=clf, cv="prefit")
 mapie_score.fit(X_cal, y_cal)
 alpha = [0.2, 0.1, 0.05]
 y_pred_score, y_ps_score = mapie_score.predict(X_test_mesh, alpha=alpha)
@@ -122,7 +123,7 @@
 # * ``y_pred_score``: represents the prediction in the test set by the base
 #   estimator.
 # * ``y_ps_score``: reprensents the prediction sets estimated by MAPIE with
-#   the "lac" method.
+#   the "lac" conformity score.
 
 
 def plot_scores(n, alphas, scores, quantiles):
@@ -240,7 +241,7 @@ def plot_coverages_widths(alpha, coverage, width, method):
 # 2. Conformal Prediction method using the cumulative softmax score
 # -----------------------------------------------------------------
 #
-# We saw in the previous section that the "lac" method is well calibrated by
+# We saw in the previous section that the "lac" conformity score is well calibrated by
 # providing accurate coverage levels. However, it tends to give null
 # prediction sets for uncertain regions, especially when the ``α``
 # value is high.
@@ -254,7 +255,7 @@ def plot_coverages_widths(alpha, coverage, width, method):
 # set after fitting MAPIE on the calibration set.
 
 mapie_aps = _MapieClassifier(
-    estimator=clf, cv="prefit", method="aps"
+    estimator=clf, cv="prefit", conformity_score=APSConformityScore()
 )
 mapie_aps.fit(X_cal, y_cal)
 alpha = [0.2, 0.1, 0.05]

mapie/classification.py

@@ -582,42 +582,6 @@ class _MapieClassifier(ClassifierMixin, BaseEstimator):
         (i.e. with fit, predict, and predict_proba methods), by default None.
         If ``None``, estimator defaults to a ``LogisticRegression`` instance.
 
-    method: Optional[str]
-        [DEPRECIATED see instead conformity_score]
-        Method to choose for prediction interval estimates.
-        Choose among:
-
-        - ``"naive"``, sum of the probabilities until the 1-alpha threshold.
-
-        - ``"lac"``, Least Ambiguous set-valued
-          Classifier. It is based on the scores
-          (i.e. 1 minus the softmax score of the true label)
-          on the calibration set. See [1] for more details.
-
-        - ``"aps"``, Adaptive Prediction
-          Sets method. It is based on the sum of the softmax outputs of the
-          labels until the true label is reached, on the calibration set.
-          See [2] for more details.
-
-        - ``"raps"``, Regularized Adaptive Prediction Sets method. It uses the
-          same technique as ``"aps"`` method but with a penalty term
-          to reduce the size of prediction sets. See [3] for more
-          details. For now, this method only works with ``"prefit"`` and
-          ``"split"`` strategies.
-
-        - ``"top_k"``, based on the sorted index of the probability of the true
-          label in the softmax outputs, on the calibration set. In case two
-          probabilities are equal, both are taken, thus, the size of some
-          prediction sets may be different from the others. See [3] for
-          more details.
-
-        - ``None``, that does not specify the method used.
-
-        In any case, the `method` parameter does not take precedence over the
-        `conformity_score` parameter to define the method used.
-
-        By default ``None``.
-
     cv: Optional[Union[int, str, BaseCrossValidator]]
         The cross-validation strategy for computing scores.
         It directly drives the distinction between jackknife and cv variants.
@@ -667,9 +631,6 @@ class _MapieClassifier(ClassifierMixin, BaseEstimator):
     conformity_score: BaseClassificationScore
         Score function that handle all that is related to conformity scores.
 
-        In any case, the `conformity_score` parameter takes precedence over the
-        `method` parameter to define the method used.
-
         By default ``None``.
 
     random_state: Optional[Union[int, RandomState]]
@@ -757,7 +718,6 @@ class _MapieClassifier(ClassifierMixin, BaseEstimator):
     def __init__(
         self,
         estimator: Optional[ClassifierMixin] = None,
-        method: Optional[str] = None,
         cv: Optional[Union[int, str, BaseCrossValidator]] = None,
         test_size: Optional[Union[int, float]] = None,
         n_jobs: Optional[int] = None,
@@ -766,7 +726,6 @@ def __init__(
         verbose: int = 0
     ) -> None:
         self.estimator = estimator
-        self.method = method
         self.cv = cv
         self.test_size = test_size
         self.n_jobs = n_jobs
@@ -862,40 +821,6 @@ def _check_fit_parameter(
     ):
         """
         Perform several checks on class parameters.
-
-        Parameters
-        ----------
-        X: ArrayLike
-            Observed values.
-
-        y: ArrayLike
-            Target values.
-
-        sample_weight: Optional[ArrayLike] of shape (n_samples,)
-            Non-null sample weights.
-
-        groups: Optional[ArrayLike] of shape (n_samples,)
-            Group labels for the samples used while splitting the dataset into
-            train/test set.
-            By default ``None``.
-
-        Returns
-        -------
-        Tuple[Optional[ClassifierMixin],
-        Optional[Union[int, str, BaseCrossValidator]],
-        ArrayLike, NDArray, NDArray, Optional[NDArray],
-        Optional[NDArray], ArrayLike]
-            Parameters checked
-
-        Raises
-        ------
-        ValueError
-            If conformity score is FittedResidualNormalizing score and method
-            is neither ``"prefit"`` or ``"split"``.
-
-        ValueError
-            If ``cv`` is `"prefit"`` or ``"split"`` and ``method`` is not
-            ``"base"``.
         """
         self._check_parameters()
         cv = _check_cv(
@@ -917,10 +842,7 @@ def _check_fit_parameter(
         self.label_encoder_ = self._get_label_encoder()
         y_enc = self.label_encoder_.transform(y)
 
-        cs_estimator = check_classification_conformity_score(
-            conformity_score=self.conformity_score,
-            method=self.method,
-        )
+        cs_estimator = check_classification_conformity_score(self.conformity_score)
         check_depreciated_size_raps(size_raps)
         cs_estimator.set_external_attributes(
             classes=self.classes_,
@@ -936,7 +858,7 @@ def _check_fit_parameter(
             )
         ):
             raise ValueError(
-                "RAPS method can only be used "
+                "RAPS conformity score can only be used "
                 "with SplitConformalClassifier."
             )