Master merge tests v0 & v1

Makefile

@@ -4,7 +4,7 @@
 
 ### Checks that are run in GitHub CI ###
 lint:
-	flake8 examples mapie notebooks tests_v1 --max-line-length=88
+	flake8 examples mapie notebooks --max-line-length=88
 
 type-check:
 	mypy mapie
@@ -14,7 +14,7 @@ coverage:
 		--cov-branch \
 		--cov=mapie \
 		--cov-report term-missing \
-		--pyargs mapie tests_v1 \
+		--pyargs mapie \
 		--cov-fail-under=100 \
 		--no-cov-on-fail \
 		--doctest-modules
@@ -37,7 +37,6 @@ all-checks:
 
 tests:
 	pytest -vs --doctest-modules mapie
-	python -m pytest -vs tests_v1
 
 clean-doc:
 	$(MAKE) clean -C doc

mapie/tests/test_common.py

@@ -4,15 +4,221 @@
 import numpy as np
 import pytest
 from sklearn.base import BaseEstimator
+from sklearn.datasets import make_regression, make_classification
+from sklearn.dummy import DummyRegressor, DummyClassifier
 from sklearn.exceptions import NotFittedError
-from sklearn.linear_model import LinearRegression, LogisticRegression
-from sklearn.model_selection import KFold
+from sklearn.linear_model import LinearRegression, LogisticRegression, QuantileRegressor
+from sklearn.model_selection import KFold, train_test_split
 from sklearn.pipeline import make_pipeline
 from sklearn.utils.validation import check_is_fitted
 
-from mapie.classification import _MapieClassifier
-from mapie.regression.regression import _MapieRegressor
-from mapie.regression.quantile_regression import _MapieQuantileRegressor
+from mapie.classification import _MapieClassifier, SplitConformalClassifier, \
+    CrossConformalClassifier
+from mapie.regression.regression import _MapieRegressor, SplitConformalRegressor, \
+    CrossConformalRegressor, JackknifeAfterBootstrapRegressor
+from mapie.regression.quantile_regression import _MapieQuantileRegressor, \
+    ConformalizedQuantileRegressor
+
+RANDOM_STATE = 1
+
+
+@pytest.fixture(scope="module")
+def dataset_regression():
+    X, y = make_regression(
+        n_samples=500, n_features=2, noise=1.0, random_state=RANDOM_STATE
+    )
+    X_train, X_conf_test, y_train, y_conf_test = train_test_split(
+        X, y, random_state=RANDOM_STATE
+    )
+    X_conformalize, X_test, y_conformalize, y_test = train_test_split(
+        X_conf_test, y_conf_test, random_state=RANDOM_STATE
+    )
+    return X_train, X_conformalize, X_test, y_train, y_conformalize, y_test
+
+
+@pytest.fixture(scope="module")
+def dataset_classification():
+    X, y = make_classification(
+        n_samples=500, n_informative=5, n_classes=4, random_state=RANDOM_STATE,
+    )
+    X_train, X_conf_test, y_train, y_conf_test = train_test_split(
+        X, y, random_state=RANDOM_STATE
+    )
+    X_conformalize, X_test, y_conformalize, y_test = train_test_split(
+        X_conf_test, y_conf_test, random_state=RANDOM_STATE
+    )
+    return X_train, X_conformalize, X_test, y_train, y_conformalize, y_test
+
+
+def test_scr_same_predictions_prefit_not_prefit(dataset_regression) -> None:
+    X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = (
+        dataset_regression)
+    regressor = LinearRegression()
+    regressor.fit(X_train, y_train)
+    scr_prefit = SplitConformalRegressor(estimator=regressor, prefit=True)
+    scr_prefit.conformalize(X_conformalize, y_conformalize)
+    predictions_scr_prefit = scr_prefit.predict_interval(X_test)
+
+    scr_not_prefit = SplitConformalRegressor(estimator=LinearRegression(), prefit=False)
+    scr_not_prefit.fit(X_train, y_train).conformalize(X_conformalize, y_conformalize)
+    predictions_scr_not_prefit = scr_not_prefit.predict_interval(X_test)
+    np.testing.assert_equal(predictions_scr_prefit, predictions_scr_not_prefit)
+
+
+@pytest.mark.parametrize(
+    "split_technique,predict_method,dataset,estimator_class",
+    [
+        (
+            SplitConformalRegressor,
+            "predict_interval",
+            "dataset_regression",
+            DummyRegressor
+        ),
+        (
+            ConformalizedQuantileRegressor,
+            "predict_interval",
+            "dataset_regression",
+            QuantileRegressor
+        ),
+        (
+            SplitConformalClassifier,
+            "predict_set",
+            "dataset_classification",
+            DummyClassifier
+        )
+    ]
+)
+class TestWrongMethodsOrderRaisesErrorForSplitTechniques:
+    def test_with_prefit_false(
+        self,
+        split_technique,
+        predict_method,
+        dataset,
+        estimator_class,
+        request
+    ):
+        dataset = request.getfixturevalue(dataset)
+        X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset
+        estimator = estimator_class()
+        technique = split_technique(estimator=estimator, prefit=False)
+
+        with pytest.raises(ValueError, match=r"call fit before calling conformalize"):
+            technique.conformalize(
+                X_conformalize,
+                y_conformalize
+            )
+
+        technique.fit(X_train, y_train)
+
+        with pytest.raises(ValueError, match=r"fit method already called"):
+            technique.fit(X_train, y_train)
+        with pytest.raises(
+            ValueError,
+            match=r"call conformalize before calling predict"
+        ):
+            technique.predict(X_test)
+
+        with pytest.raises(
+            ValueError,
+            match=f"call conformalize before calling {predict_method}"
+        ):
+            getattr(technique, predict_method)(X_test)
+
+        technique.conformalize(X_conformalize, y_conformalize)
+
+        with pytest.raises(ValueError, match=r"conformalize method already called"):
+            technique.conformalize(X_conformalize, y_conformalize)
+
+    def test_with_prefit_true(
+        self,
+        split_technique,
+        predict_method,
+        dataset,
+        estimator_class,
+        request
+    ):
+        dataset = request.getfixturevalue(dataset)
+        X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset
+        estimator = estimator_class()
+        estimator.fit(X_train, y_train)
+
+        if split_technique == ConformalizedQuantileRegressor:
+            technique = split_technique(estimator=[estimator] * 3, prefit=True)
+        else:
+            technique = split_technique(estimator=estimator, prefit=True)
+
+        with pytest.raises(ValueError, match=r"The fit method must be skipped"):
+            technique.fit(X_train, y_train)
+        with pytest.raises(
+            ValueError,
+            match=r"call conformalize before calling predict"
+        ):
+            technique.predict(X_test)
+
+        with pytest.raises(
+            ValueError,
+            match=f"call conformalize before calling {predict_method}"
+        ):
+            getattr(technique, predict_method)(X_test)
+
+        technique.conformalize(X_conformalize, y_conformalize)
+
+        with pytest.raises(ValueError, match=r"conformalize method already called"):
+            technique.conformalize(X_conformalize, y_conformalize)
+
+
+@pytest.mark.parametrize(
+    "cross_technique,predict_method,dataset,estimator_class",
+    [
+        (
+            CrossConformalRegressor,
+            "predict_interval",
+            "dataset_regression",
+            DummyRegressor
+        ),
+        (
+            JackknifeAfterBootstrapRegressor,
+            "predict_interval",
+            "dataset_regression",
+            DummyRegressor
+        ),
+        (
+            CrossConformalClassifier,
+            "predict_set",
+            "dataset_classification",
+            DummyClassifier
+        ),
+    ]
+)
+class TestWrongMethodsOrderRaisesErrorForCrossTechniques:
+    def test_wrong_methods_order(
+        self,
+        cross_technique,
+        predict_method,
+        dataset,
+        estimator_class,
+        request
+    ):
+        dataset = request.getfixturevalue(dataset)
+        X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset
+        technique = cross_technique(estimator=estimator_class())
+
+        with pytest.raises(
+            ValueError,
+            match=r"call fit_conformalize before calling predict"
+        ):
+            technique.predict(X_test)
+        with pytest.raises(
+            ValueError,
+            match=f"call fit_conformalize before calling {predict_method}"
+        ):
+            getattr(technique, predict_method)(X_test)
+
+        technique.fit_conformalize(X_conformalize, y_conformalize)
+
+        with pytest.raises(ValueError, match=r"fit_conformalize method already called"):
+            technique.fit_conformalize(X_conformalize, y_conformalize)
+
 
 X_toy = np.arange(18).reshape(-1, 1)
 y_toy = np.array(

mapie/tests/test_conformity_scores_utils.py

@@ -3,9 +3,37 @@
 import numpy as np
 import pytest
 
+from mapie.conformity_scores import AbsoluteConformityScore, BaseRegressionScore, \
+    GammaConformityScore, LACConformityScore, BaseClassificationScore, \
+    TopKConformityScore
 from mapie.conformity_scores.sets.utils import get_true_label_position
 from numpy.typing import NDArray
 
+from mapie.conformity_scores.utils import check_and_select_conformity_score
+
+
+class TestCheckAndSelectConformityScore:
+
+    @pytest.mark.parametrize(
+        "score, score_type, expected_class", [
+            (AbsoluteConformityScore(), BaseRegressionScore, AbsoluteConformityScore),
+            ("gamma", BaseRegressionScore, GammaConformityScore),
+            (LACConformityScore(), BaseClassificationScore, LACConformityScore),
+            ("top_k", BaseClassificationScore, TopKConformityScore),
+        ]
+    )
+    def test_with_valid_inputs(self, score, score_type, expected_class):
+        result = check_and_select_conformity_score(score, score_type)
+        assert isinstance(result, expected_class)
+
+    @pytest.mark.parametrize(
+        "score_type", [BaseRegressionScore, BaseClassificationScore]
+    )
+    def test_with_invalid_input(self, score_type):
+        with pytest.raises(ValueError):
+            check_and_select_conformity_score("I'm not a valid input :(", score_type)
+
+
 Y_TRUE_PROBA_PLACE = [
     [
         np.array([2, 0]),