VTA comments taken into account

Author: LacombeLouis

examples/regression/1-quickstart/plot_prefit.py

@@ -12,7 +12,7 @@
 In this example, we first fit a neural network on the training set. We
 then compute residuals on a validation set with the `cv="prefit"` parameter.
 Finally, we evaluate the model with prediction intervals on a testing set.
-We will also show how to use the prefit method in the comformalized quantile
+We will also show how to use the prefit method in the conformalized quantile
 regressor.
 """
 
@@ -23,12 +23,13 @@
 import scipy
 from sklearn.model_selection import train_test_split
 from sklearn.neural_network import MLPRegressor
+import warnings
 
 from mapie.regression import MapieRegressor
 from mapie.quantile_regression import MapieQuantileRegressor
 from mapie.metrics import regression_coverage_score
 from mapie._typing import NDArray
-import warnings
+
 warnings.filterwarnings("ignore")
 
 alpha = 0.1
@@ -38,8 +39,8 @@
 # -----------------------------------------------------------------------------
 #
 # We start by defining a function that we will use to generate data. We then
-# add random noise y values. Then we split the dataset to have a training,
-# calibration and test set.
+# add random noise dependent variable values. Then we split the dataset to have
+# a training, calibration and test set.
 
 
 def f(x: NDArray) -> NDArray:
@@ -67,12 +68,12 @@ def f(x: NDArray) -> NDArray:
 # -----------------------------------------------------------------------------
 #
 # For this example, we will train a MLPRegressor for
-# :class:`mapie.regression.MapieRegressor` and multiple LGBMRegressor in the
-# with a quantile objective as this is a requirement to perform conformalized
+# :class:`mapie.regression.MapieRegressor` and multiple LGBMRegressor with a
+# quantile objective as this is a requirement to perform conformalized
 # quantile regression using
 # :class:`mapie.quanitle_regression.MapieQuantileRegressor`. Note that the
 # three estimators need to be trained at quantile values of
-# $(\alpha/2, 1-(\alpha/2), 0.5)$.
+# :math:`(\alpha/2, 1-(\alpha/2), 0.5)`.
 
 
 # Train a MLPRegressor for MapieRegressor
@@ -81,7 +82,7 @@ def f(x: NDArray) -> NDArray:
 
 # Train LGBMRegressor models for MapieQuantileRegressor
 list_estimators_cqr = []
-for alpha_ in [alpha/2, (1-(alpha/2)), 0.5]:
+for alpha_ in [alpha / 2, (1 - (alpha / 2)), 0.5]:
     estimator_ = LGBMRegressor(
         objective='quantile',
         alpha=alpha_,
@@ -94,8 +95,8 @@ def f(x: NDArray) -> NDArray:
 # 3. Using MAPIE to calibrate the models
 # -----------------------------------------------------------------------------
 #
-# We will now proceed to calibrate the models using MAPIE. This means using
-# the `cv="prefit"` so that we use the models that we already trained prior.
+# We will now proceed to calibrate the models using MAPIE. To this aim, we set
+# `cv="prefit"` so that we use the models that we already trained prior.
 # We then precict using the test set and evaluate its coverage.
 
 
@@ -124,7 +125,7 @@ def f(x: NDArray) -> NDArray:
 # 4. Plots
 # -----------------------------------------------------------------------------
 #
-# In order to view the results shown above, we will plot each othe predictions
+# In order to view the results shown above, we will plot each other predictions
 # with their prediction interval. The multi-layer perceptron (MLP) with
 # :class:`mapie.regression.MapieRegressor` and LGBMRegressor with
 # :class:`mapie.quantile_regression.MapieQuantileRegressor`.

mapie/tests/test_quantile_regression.py

@@ -519,7 +519,7 @@ def test_linear_regression_results(strategy: str) -> None:
     np.testing.assert_allclose(coverage, COVERAGES[strategy], rtol=1e-2)
 
 
-@pytest.mark.parametrize("estimator", [-1, 3, KFold(), LeaveOneOut()])
+@pytest.mark.parametrize("estimator", [-1, 3, 0.2])
 def test_quantile_prefit_non_list(estimator: Any) -> None:
     """
     Test that there is a list of estimators provided when cv='prefit'
@@ -633,12 +633,14 @@ def test_warning_alpha_prefit() -> None:
         )
 
 
-def test_prefit_and_non_prefit_equal() -> None:
+@pytest.mark.parametrize("alpha", [0.05, 0.1, 0.2, 0.3])
+def test_prefit_and_non_prefit_equal(alpha: float) -> None:
     """
-    Check that the user is warned that the alphas need to be correctly set.
+    Check that when using prefit and not prefit, the same values
+    are found.
     """
     list_estimators = []
-    alphas_ = [0.15, 0.85, 0.5]
+    alphas_ = [alpha/2, 1-(alpha/2), 0.5]
     for alpha_ in alphas_:
         est = clone(qt)
         params = {"quantile": alpha_}
@@ -648,14 +650,14 @@ def test_prefit_and_non_prefit_equal() -> None:
     mapie_reg_prefit = MapieQuantileRegressor(
         estimator=list_estimators,
         cv="prefit",
-        alpha=0.3
+        alpha=alpha
     )
     mapie_reg_prefit.fit(X_calib, y_calib)
     y_pred_prefit, y_pis_prefit = mapie_reg_prefit.predict(X)
 
     mapie_reg = MapieQuantileRegressor(
         estimator=qt,
-        alpha=0.3
+        alpha=alpha
     )
     mapie_reg.fit(X_train, y_train, X_calib=X_calib, y_calib=y_calib)
     y_pred, y_pis = mapie_reg.predict(X)
@@ -664,6 +666,52 @@ def test_prefit_and_non_prefit_equal() -> None:
     np.testing.assert_allclose(y_pis_prefit, y_pis)
 
 
+@pytest.mark.parametrize("alpha", [0.05, 0.1, 0.2, 0.3])
+def test_prefit_different_type_list_tuple_array(alpha: float) -> None:
+    """
+    Check that as long as the estimators are given in a Iterable object,
+    the we have the same results for each.}
+    """
+    list_estimators = []
+    alphas_ = [alpha/2, 1-(alpha/2), 0.5]
+    for alpha_ in alphas_:
+        est = clone(qt)
+        params = {"quantile": alpha_}
+        est.set_params(**params)
+        est.fit(X_train, y_train)
+        list_estimators.append(est)
+
+    mapie_reg_prefit_list = MapieQuantileRegressor(
+        estimator=list_estimators,
+        cv="prefit",
+        alpha=alpha
+    )
+    mapie_reg_prefit_list.fit(X_calib, y_calib)
+    y_pred_prefit_list, y_pis_prefit_list = mapie_reg_prefit_list.predict(X)
+
+    mapie_reg_prefit_tuple = MapieQuantileRegressor(
+        estimator=tuple(list_estimators),
+        cv="prefit",
+        alpha=alpha
+    )
+    mapie_reg_prefit_tuple.fit(X_calib, y_calib)
+    y_pred_prefit_tuple, y_pis_prefit_tuple = mapie_reg_prefit_tuple.predict(X)
+
+    mapie_reg_prefit_array = MapieQuantileRegressor(
+        estimator=np.array(list_estimators),
+        cv="prefit",
+        alpha=alpha
+    )
+    mapie_reg_prefit_array.fit(X_calib, y_calib)
+    y_pred_prefit_array, y_pis_prefit_array = mapie_reg_prefit_array.predict(X)
+
+    np.testing.assert_allclose(y_pred_prefit_list, y_pred_prefit_tuple)
+    np.testing.assert_allclose(y_pis_prefit_list, y_pis_prefit_tuple)
+
+    np.testing.assert_allclose(y_pred_prefit_list, y_pred_prefit_array)
+    np.testing.assert_allclose(y_pis_prefit_list, y_pis_prefit_array)
+
+
 @pytest.mark.parametrize("estimator", ESTIMATOR)
 def test_pipeline_compatibility(estimator: RegressorMixin) -> None:
     """Check that MAPIE works on pipeline based on pandas dataframes"""

mapie/tests/test_utils.py

@@ -242,7 +242,7 @@ def test_alpha_in_predict() -> None:
         mapie_reg.predict(X, ensemble=True)
 
 
-@pytest.mark.parametrize("estimator", [-1, 3])
+@pytest.mark.parametrize("estimator", [-1, 3, 0.2])
 def test_quantile_prefit_non_iterable(estimator: Any) -> None:
     """
     Test that there is a list of estimators provided when cv='prefit'