ADD: coverage validity test

Author: Thibault Cordier

mapie/tests/test_regression.py

@@ -259,6 +259,38 @@ def test_predict_output_shape(
     assert y_pis.shape == (X.shape[0], 2, n_alpha)
 
 
+@pytest.mark.parametrize("delta", [0.5, 0.6, 0.7, 0.8])
+@pytest.mark.parametrize("n_calib", [10, 20, 50, 100])
+def test_coverage_validity(delta: float, n_calib: int) -> None:
+    """
+    Test that the prefit method provides valid coverage
+    for different calibration data sizes and coverage targets.
+    """
+    n_split, n_train, n_test = 1000, 100, 100
+    n_all = n_train + n_calib + n_test
+    X, y = make_regression(n_all, random_state=random_state)
+
+    X_train, X_cal_test, y_train, y_cal_test = \
+        train_test_split(X, y, train_size=n_train, random_state=random_state)
+
+    model = LinearRegression()
+    model.fit(X_train, y_train)
+
+    coverage_list = []
+    for _ in range(n_split):
+        mapie_reg = MapieRegressor(estimator=model, method="base", cv="prefit")
+        X_cal, X_test, y_cal, y_test = \
+            train_test_split(X_cal_test, y_cal_test, test_size=n_test)
+        mapie_reg.fit(X_cal, y_cal)
+        _, y_pis = mapie_reg.predict(X_test, alpha=1-delta)
+        coverage = \
+            regression_coverage_score(y_test,  y_pis[:, 0, 0], y_pis[:, 1, 0])
+        coverage_list.append(coverage)
+
+    mean_coverage = np.mean(coverage_list)
+    np.testing.assert_array_less(delta, mean_coverage)
+
+
 def test_same_results_prefit_split() -> None:
     """
     Test checking that if split and prefit method have exactly