From 92cd411ec6ead9acc3e1ffb2dd072516277fbc8e Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Thu, 31 Jul 2025 17:19:11 +0200 Subject: [PATCH 01/14] Merge tests v0 and v1 --- mapie/tests/test_classification.py | 145 ++- mapie/tests/test_conformity_scores_utils.py | 27 + mapie/tests/test_non_regression_v0_to_v1.py | 1038 +++++++++++++++++ mapie/tests/test_regression.py | 203 +++- mapie/tests/test_utils.py | 331 +++++- tests_v1/test_functional/test_functional.py | 212 ---- .../test_non_regression_classification.py | 2 +- .../test_non_regression_regression.py | 2 +- tests_v1/test_functional/utils.py | 77 -- .../test_unit/test_conformity_scores_utils.py | 37 - tests_v1/test_unit/test_regression.py | 28 - tests_v1/test_unit/test_utils.py | 266 ----- 12 files changed, 1741 insertions(+), 627 deletions(-) create mode 100644 mapie/tests/test_non_regression_v0_to_v1.py delete mode 100644 tests_v1/test_functional/test_functional.py delete mode 100644 tests_v1/test_functional/utils.py delete mode 100644 tests_v1/test_unit/test_conformity_scores_utils.py delete mode 100644 tests_v1/test_unit/test_regression.py delete mode 100644 tests_v1/test_unit/test_utils.py diff --git a/mapie/tests/test_classification.py b/mapie/tests/test_classification.py index 905e7a4e2..a5e920ec7 100644 --- a/mapie/tests/test_classification.py +++ b/mapie/tests/test_classification.py @@ -24,7 +24,8 @@ from typing_extensions import TypedDict from numpy.typing import ArrayLike, NDArray -from mapie.classification import _MapieClassifier +from mapie.classification import _MapieClassifier, SplitConformalClassifier, \ + CrossConformalClassifier from mapie.conformity_scores import ( LACConformityScore, RAPSConformityScore, @@ -2029,3 +2030,145 @@ def test_using_one_predict_parameter_into_fit_but_not_in_predict() -> None: r"is used in the predict method as called in the fit." )): mapie_fitted.predict(X_test) + + +@pytest.fixture(scope="module") +def dataset_classification(): + X, y = make_classification( + n_samples=500, n_informative=5, n_classes=4, random_state=random_state, + ) + X_train, X_conf_test, y_train, y_conf_test = train_test_split( + X, y, random_state=random_state + ) + X_conformalize, X_test, y_conformalize, y_test = train_test_split( + X_conf_test, y_conf_test, random_state=random_state + ) + return X_train, X_conformalize, X_test, y_train, y_conformalize, y_test + + +@pytest.mark.parametrize( + "split_technique,predict_method,dataset,estimator_class", + [ + ( + SplitConformalClassifier, + "predict_set", + "dataset_classification", + DummyClassifier + ) + ] +) +class TestWrongMethodsOrderRaisesErrorForSplitTechniquesClassification: + def test_with_prefit_false( + self, + split_technique, + predict_method, + dataset, + estimator_class, + request + ): + dataset = request.getfixturevalue(dataset) + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset + estimator = estimator_class() + technique = split_technique(estimator=estimator, prefit=False) + + with pytest.raises(ValueError, match=r"call fit before calling conformalize"): + technique.conformalize( + X_conformalize, + y_conformalize + ) + + technique.fit(X_train, y_train) + + with pytest.raises(ValueError, match=r"fit method already called"): + technique.fit(X_train, y_train) + with pytest.raises( + ValueError, + match=r"call conformalize before calling predict" + ): + technique.predict(X_test) + + with pytest.raises( + ValueError, + match=f"call conformalize before calling {predict_method}" + ): + getattr(technique, predict_method)(X_test) + + technique.conformalize(X_conformalize, y_conformalize) + + with pytest.raises(ValueError, match=r"conformalize method already called"): + technique.conformalize(X_conformalize, y_conformalize) + + def test_with_prefit_true( + self, + split_technique, + predict_method, + dataset, + estimator_class, + request + ): + dataset = request.getfixturevalue(dataset) + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset + estimator = estimator_class() + estimator.fit(X_train, y_train) + + technique = split_technique(estimator=estimator, prefit=True) + + with pytest.raises(ValueError, match=r"The fit method must be skipped"): + technique.fit(X_train, y_train) + with pytest.raises( + ValueError, + match=r"call conformalize before calling predict" + ): + technique.predict(X_test) + + with pytest.raises( + ValueError, + match=f"call conformalize before calling {predict_method}" + ): + getattr(technique, predict_method)(X_test) + + technique.conformalize(X_conformalize, y_conformalize) + + with pytest.raises(ValueError, match=r"conformalize method already called"): + technique.conformalize(X_conformalize, y_conformalize) + + +@pytest.mark.parametrize( + "cross_technique,predict_method,dataset,estimator_class", + [ + ( + CrossConformalClassifier, + "predict_set", + "dataset_classification", + DummyClassifier + ) + ] +) +class TestWrongMethodsOrderRaisesErrorForCrossTechniques: + def test_wrong_methods_order( + self, + cross_technique, + predict_method, + dataset, + estimator_class, + request + ): + dataset = request.getfixturevalue(dataset) + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset + technique = cross_technique(estimator=estimator_class()) + + with pytest.raises( + ValueError, + match=r"call fit_conformalize before calling predict" + ): + technique.predict(X_test) + with pytest.raises( + ValueError, + match=f"call fit_conformalize before calling {predict_method}" + ): + getattr(technique, predict_method)(X_test) + + technique.fit_conformalize(X_conformalize, y_conformalize) + + with pytest.raises(ValueError, match=r"fit_conformalize method already called"): + technique.fit_conformalize(X_conformalize, y_conformalize) diff --git a/mapie/tests/test_conformity_scores_utils.py b/mapie/tests/test_conformity_scores_utils.py index 4636d2396..df71c06ab 100644 --- a/mapie/tests/test_conformity_scores_utils.py +++ b/mapie/tests/test_conformity_scores_utils.py @@ -3,9 +3,14 @@ import numpy as np import pytest +from mapie.conformity_scores import AbsoluteConformityScore, BaseRegressionScore, \ + GammaConformityScore, LACConformityScore, BaseClassificationScore, \ + TopKConformityScore from mapie.conformity_scores.sets.utils import get_true_label_position from numpy.typing import NDArray +from mapie.conformity_scores.utils import check_and_select_conformity_score + Y_TRUE_PROBA_PLACE = [ [ np.array([2, 0]), @@ -50,3 +55,25 @@ def test_get_true_label_position( found_place = get_true_label_position(y_pred_proba, y_true) assert (found_place == place).all() + + +class TestCheckAndSelectConformityScore: + + @pytest.mark.parametrize( + "score, score_type, expected_class", [ + (AbsoluteConformityScore(), BaseRegressionScore, AbsoluteConformityScore), + ("gamma", BaseRegressionScore, GammaConformityScore), + (LACConformityScore(), BaseClassificationScore, LACConformityScore), + ("top_k", BaseClassificationScore, TopKConformityScore), + ] + ) + def test_with_valid_inputs(self, score, score_type, expected_class): + result = check_and_select_conformity_score(score, score_type) + assert isinstance(result, expected_class) + + @pytest.mark.parametrize( + "score_type", [BaseRegressionScore, BaseClassificationScore] + ) + def test_with_invalid_input(self, score_type): + with pytest.raises(ValueError): + check_and_select_conformity_score("I'm not a valid input :(", score_type) diff --git a/mapie/tests/test_non_regression_v0_to_v1.py b/mapie/tests/test_non_regression_v0_to_v1.py new file mode 100644 index 000000000..bea385fa8 --- /dev/null +++ b/mapie/tests/test_non_regression_v0_to_v1.py @@ -0,0 +1,1038 @@ +from typing import Type, Union, Dict, Optional + +import numpy as np +import pytest +from numpy._typing import ArrayLike, NDArray +from numpy.random import RandomState +from sklearn.compose import TransformedTargetRegressor +from sklearn.datasets import make_classification, make_regression +from sklearn.ensemble import RandomForestClassifier, GradientBoostingRegressor +from sklearn.linear_model import LogisticRegression, LinearRegression, QuantileRegressor +from sklearn.model_selection import LeaveOneOut, GroupKFold, train_test_split + +from mapie.classification import _MapieClassifier, SplitConformalClassifier, \ + CrossConformalClassifier +from mapie.conformity_scores import LACConformityScore, TopKConformityScore, \ + APSConformityScore, RAPSConformityScore, AbsoluteConformityScore, \ + GammaConformityScore, ResidualNormalisedScore +from mapie.regression import CrossConformalRegressor, JackknifeAfterBootstrapRegressor +from mapie.regression.quantile_regression import _MapieQuantileRegressor, \ + ConformalizedQuantileRegressor +from mapie.regression.regression import _MapieRegressor, SplitConformalRegressor +from mapie.subsample import Subsample +from mapie.tests.test_utils import train_test_split_shuffle, \ + DummyClassifierWithFitAndPredictParams, filter_params + +RANDOM_STATE = 1 +K_FOLDS = 3 +N_BOOTSTRAPS = 30 +N_SAMPLES = 200 +N_GROUPS = 5 + + +@pytest.fixture(scope="module") +def dataset(): + X, y = make_classification( + n_samples=1000, + n_informative=5, + n_classes=4, + random_state=RANDOM_STATE + ) + sample_weight = RandomState(RANDOM_STATE).random(len(X)) + groups = np.array([i % 5 for i in range(len(X))]) + + ( + X_train, + X_conformalize, + y_train, + y_conformalize, + sample_weight_train, + sample_weight_conformalize, + ) = train_test_split_shuffle( + X, y, random_state=RANDOM_STATE, sample_weight=sample_weight + ) + + return { + "X": X, + "y": y, + "sample_weight": sample_weight, + "groups": groups, + "X_train": X_train, + "X_conformalize": X_conformalize, + "y_train": y_train, + "y_conformalize": y_conformalize, + "sample_weight_train": sample_weight_train, + "sample_weight_conformalize": sample_weight_conformalize, + } + + +@pytest.fixture() +def params_split_test_1(): + return { + "v1": { + "__init__": { + "estimator": LogisticRegression(), + }, + }, + "v0": { + "__init__": { + "estimator": LogisticRegression(), + "conformity_score": LACConformityScore(), + "cv": "prefit" + }, + "predict": { + "alpha": 0.1, + }}} + + +@pytest.fixture() +def params_split_test_2(): + return { + "v1": { + "__init__": { + "estimator": DummyClassifierWithFitAndPredictParams(), + "confidence_level": 0.8, + "prefit": False, + "conformity_score": "top_k", + "random_state": RANDOM_STATE, + }, + "fit": { + "fit_params": {"dummy_fit_param": True}, + }, + "conformalize": { + "predict_params": {"dummy_predict_param": True}, + }}, + "v0": { + "__init__": { + "estimator": DummyClassifierWithFitAndPredictParams(), + "conformity_score": TopKConformityScore(), + "cv": "split", + "random_state": RANDOM_STATE, + }, + "fit": { + "fit_params": {"dummy_fit_param": True}, + "predict_params": {"dummy_predict_param": True}, + }, + "predict": { + "alpha": 0.2, + "dummy_predict_param": True, + }}} + + +@pytest.fixture() +def params_split_test_3(dataset): + return { + "v1": { + "__init__": { + "estimator": RandomForestClassifier(random_state=RANDOM_STATE), + "confidence_level": [0.8, 0.9], + "prefit": False, + "conformity_score": "aps", + "random_state": RANDOM_STATE, + }, + "fit": { + "fit_params": {"sample_weight": dataset["sample_weight_train"]}, + }, + "predict_set": { + "conformity_score_params": {"include_last_label": False} + }}, + "v0": { + "__init__": { + "estimator": RandomForestClassifier(random_state=RANDOM_STATE), + "conformity_score": APSConformityScore(), + "cv": "split", + "random_state": RANDOM_STATE, + }, + "fit": { + "sample_weight": dataset["sample_weight"], + }, + "predict": { + "alpha": [0.2, 0.1], + "include_last_label": False, + }}} + + +@pytest.fixture() +def params_split_test_4(): + return { + "v1": { + "__init__": { + "estimator": LogisticRegression(), + "conformity_score": "raps", + "random_state": RANDOM_STATE, + }}, + "v0": { + "__init__": { + "estimator": LogisticRegression(), + "conformity_score": RAPSConformityScore(), + "cv": "prefit", + "random_state": RANDOM_STATE, + }, + "predict": { + "alpha": 0.1, + }}} + + +@pytest.fixture() +def params_split_test_5(): + return { + "v1": { + "__init__": { + "estimator": LogisticRegression(), + "conformity_score": RAPSConformityScore(size_raps=0.4), + "random_state": RANDOM_STATE, + }}, + "v0": { + "__init__": { + "estimator": LogisticRegression(), + "conformity_score": RAPSConformityScore(size_raps=0.4), + "cv": "prefit", + "random_state": RANDOM_STATE, + }, + "predict": { + "alpha": 0.1, + }}} + + +@pytest.mark.parametrize( + "params_", [ + "params_split_test_1", + "params_split_test_2", + "params_split_test_3", + "params_split_test_4", + "params_split_test_5", + ] +) +def test_split_classification(dataset, params_, request): + X, y, X_train, X_conformalize, y_train, y_conformalize = ( + dataset["X"], + dataset["y"], + dataset["X_train"], + dataset["X_conformalize"], + dataset["y_train"], + dataset["y_conformalize"], + ) + + params = extract_params(request.getfixturevalue(params_)) + + prefit = params["v1_init"].get("prefit", True) + + if prefit: + params["v0_init"]["estimator"].fit(X_train, y_train) + params["v1_init"]["estimator"].fit(X_train, y_train) + + v0 = _MapieClassifier(**params["v0_init"]) + v1 = SplitConformalClassifier(**params["v1_init"]) + + if prefit: + v0.fit(X_conformalize, y_conformalize, **params["v0_fit"]) + else: + v0.fit(X, y, **params["v0_fit"]) + v1.fit(X_train, y_train, **params["v1_fit"]) + v1.conformalize(X_conformalize, y_conformalize, **params["v1_conformalize"]) + + v0_preds, v0_pred_sets = v0.predict(X_conformalize, **params["v0_predict"]) + v1_preds, v1_pred_sets = v1.predict_set(X_conformalize, **params["v1_predict_set"]) + + v1_preds_using_predict: ArrayLike = v1.predict(X_conformalize) + + np.testing.assert_array_equal(v0_preds, v1_preds) + np.testing.assert_array_equal(v0_pred_sets, v1_pred_sets) + np.testing.assert_array_equal(v1_preds_using_predict, v1_preds) + + n_confidence_level = get_number_of_confidence_levels(params["v1_init"]) + + assert v1_pred_sets.shape == ( + len(X_conformalize), + len(np.unique(y)), + n_confidence_level, + ) + + +@pytest.fixture() +def params_cross_test_1(dataset): + return { + "v1": { + "__init__": { + "estimator": LogisticRegression(), + "confidence_level": 0.8, + "conformity_score": "lac", + "cv": 4, + "random_state": RANDOM_STATE, + }, + "fit_conformalize": { + "fit_params": {"sample_weight": dataset["sample_weight"]}, + }, + }, + "v0": { + "__init__": { + "estimator": LogisticRegression(), + "conformity_score": LACConformityScore(), + "cv": 4, + "random_state": RANDOM_STATE, + }, + "fit": { + "sample_weight": dataset["sample_weight"], + }, + "predict": { + "alpha": 0.2, + }}} + + +@pytest.fixture() +def params_cross_test_2(): + return { + "v1": { + "__init__": { + "estimator": DummyClassifierWithFitAndPredictParams(), + "confidence_level": [0.9, 0.8], + "conformity_score": "aps", + "cv": LeaveOneOut(), + "random_state": RANDOM_STATE, + }, + "fit_conformalize": { + "predict_params": {"dummy_predict_param": True}, + }, + "predict_set": { + "conformity_score_params": {"include_last_label": False} + }, + }, + "v0": { + "__init__": { + "estimator": DummyClassifierWithFitAndPredictParams(), + "conformity_score": APSConformityScore(), + "cv": LeaveOneOut(), + "random_state": RANDOM_STATE, + }, + "fit": { + "predict_params": {"dummy_predict_param": True}, + }, + "predict": { + "alpha": [0.1, 0.2], + "include_last_label": False, + "dummy_predict_param": True, + }}} + + +@pytest.fixture() +def params_cross_test_3(dataset): + return { + "v1": { + "__init__": { + "estimator": DummyClassifierWithFitAndPredictParams(), + "cv": GroupKFold(), + "random_state": RANDOM_STATE, + }, + "fit_conformalize": { + "groups": dataset["groups"], + "fit_params": {"dummy_fit_param": True}, + }, + "predict_set": { + "agg_scores": "crossval", + }, + }, + "v0": { + "__init__": { + "estimator": DummyClassifierWithFitAndPredictParams(), + "cv": GroupKFold(), + "random_state": RANDOM_STATE, + }, + "fit": { + "groups": dataset["groups"], + "fit_params": {"dummy_fit_param": True}, + }, + "predict": { + "alpha": 0.1, + "agg_scores": "crossval", + }}} + + +@pytest.fixture() +def params_cross_test_4(): + return { + "v1": { + "__init__": { + "estimator": RandomForestClassifier(random_state=RANDOM_STATE), + "confidence_level": 0.7, + "conformity_score": LACConformityScore(), + "random_state": RANDOM_STATE, + }, + }, + "v0": { + "__init__": { + "estimator": RandomForestClassifier(random_state=RANDOM_STATE), + "cv": 5, + "random_state": RANDOM_STATE, + }, + "predict": { + "alpha": 0.3, + }}} + + +@pytest.mark.parametrize( + "params_", [ + "params_cross_test_1", + "params_cross_test_2", + "params_cross_test_3", + "params_cross_test_4", + ] +) +def test_cross_classification(dataset, params_, request): + X, y = dataset["X"], dataset["y"] + + params = extract_params(request.getfixturevalue(params_)) + + v0 = _MapieClassifier(**params["v0_init"]) + v1 = CrossConformalClassifier(**params["v1_init"]) + + v0.fit(X, y, **params["v0_fit"]) + v1.fit_conformalize(X, y, **params["v1_fit_conformalize"]) + + v0_preds, v0_pred_sets = v0.predict(X, **params["v0_predict"]) + v1_preds, v1_pred_sets = v1.predict_set(X, **params["v1_predict_set"]) + + v1_preds_using_predict: ArrayLike = v1.predict(X) + + np.testing.assert_array_equal(v0_preds, v1_preds) + np.testing.assert_array_equal(v0_pred_sets, v1_pred_sets) + np.testing.assert_array_equal(v1_preds_using_predict, v1_preds) + + n_confidence_level = get_number_of_confidence_levels(params["v1_init"]) + assert v1_pred_sets.shape == ( + len(X), + len(np.unique(y)), + n_confidence_level, + ) + + +def extract_params(params): + return { + "v0_init": params["v0"].get("__init__", {}), + "v0_fit": params["v0"].get("fit", {}), + "v0_predict": params["v0"].get("predict", {}), + "v1_init": params["v1"].get("__init__", {}), + "v1_fit": params["v1"].get("fit", {}), + "v1_conformalize": params["v1"].get("conformalize", {}), + "v1_predict_set": params["v1"].get("predict_set", {}), + "v1_fit_conformalize": params["v1"].get("fit_conformalize", {}) + } + + +def get_number_of_confidence_levels(v1_init_params): + confidence_level = v1_init_params.get("confidence_level", 0.9) + return 1 if isinstance(confidence_level, float) else len(confidence_level) + + +X, y_signed = make_regression( + n_samples=N_SAMPLES, + n_features=10, + noise=1.0, + random_state=RANDOM_STATE +) +y = np.abs(y_signed) +sample_weight = RandomState(RANDOM_STATE).random(len(X)) +groups = [j for j in range(N_GROUPS) for i in range((N_SAMPLES//N_GROUPS))] +positive_predictor = TransformedTargetRegressor( + regressor=LinearRegression(), + func=lambda y_: np.log(y_ + 1), + inverse_func=lambda X_: np.exp(X_) - 1 +) + +sample_weight_train = train_test_split( + X, + y, + sample_weight, + test_size=0.4, + random_state=RANDOM_STATE +)[-2] + + +params_test_cases_cross = [ + { + "v1": { + "class": CrossConformalRegressor, + "__init__": { + "confidence_level": 0.8, + "conformity_score": "absolute", + "cv": 4, + "method": "base", + "random_state": RANDOM_STATE, + }, + "fit_conformalize": { + "fit_params": {"sample_weight": sample_weight}, + }, + "predict_interval": { + "aggregate_predictions": "median", + }, + "predict": { + "aggregate_predictions": "median", + } + }, + "v0": { + "__init__": { + "conformity_score": AbsoluteConformityScore(), + "cv": 4, + "method": "base", + "random_state": RANDOM_STATE, + "agg_function": "median", + }, + "fit": { + "sample_weight": sample_weight, + }, + "predict": { + "alpha": 0.2, + "ensemble": True, + }, + }, + }, + { + "v1": { + "class": CrossConformalRegressor, + "__init__": { + "estimator": positive_predictor, + "confidence_level": [0.5, 0.5], + "conformity_score": "gamma", + "cv": LeaveOneOut(), + "method": "plus", + "random_state": RANDOM_STATE, + }, + "predict_interval": { + "minimize_interval_width": True, + }, + }, + "v0": { + "__init__": { + "estimator": positive_predictor, + "conformity_score": GammaConformityScore(), + "cv": LeaveOneOut(), + "agg_function": "mean", + "method": "plus", + "random_state": RANDOM_STATE, + }, + "predict": { + "alpha": [0.5, 0.5], + "optimize_beta": True, + "ensemble": True, + }, + }, + }, + { + "v1": { + "class": CrossConformalRegressor, + "__init__": { + "cv": GroupKFold(), + "method": "minmax", + "random_state": RANDOM_STATE, + }, + "fit_conformalize": { + "groups": groups, + }, + "predict_interval": { + "allow_infinite_bounds": True, + "aggregate_predictions": None, + }, + "predict": { + "aggregate_predictions": None, + }, + }, + "v0": { + "__init__": { + "cv": GroupKFold(), + "method": "minmax", + "random_state": RANDOM_STATE, + }, + "fit": { + "groups": groups, + }, + "predict": { + "alpha": 0.1, + "allow_infinite_bounds": True, + }, + }, + }, +] + +params_test_cases_jackknife = [ + { + "v1": { + "class": JackknifeAfterBootstrapRegressor, + "__init__": { + "confidence_level": 0.8, + "conformity_score": "absolute", + "resampling": Subsample( + n_resamplings=15, random_state=RANDOM_STATE + ), + "aggregation_method": "median", + "method": "plus", + "random_state": RANDOM_STATE, + }, + "fit_conformalize": { + "fit_params": {"sample_weight": sample_weight}, + }, + }, + "v0": { + "__init__": { + "conformity_score": AbsoluteConformityScore(), + "cv": Subsample(n_resamplings=15, random_state=RANDOM_STATE), + "agg_function": "median", + "method": "plus", + "random_state": RANDOM_STATE, + }, + "fit": { + "sample_weight": sample_weight, + }, + "predict": { + "alpha": 0.2, + "ensemble": True, + }, + }, + }, + { + "v1": { + "class": JackknifeAfterBootstrapRegressor, + "__init__": { + "estimator": positive_predictor, + "confidence_level": [0.5, 0.5], + "aggregation_method": "mean", + "conformity_score": "gamma", + "resampling": Subsample( + n_resamplings=20, + replace=True, + random_state=RANDOM_STATE + ), + "method": "plus", + "random_state": RANDOM_STATE, + }, + "predict_interval": { + "minimize_interval_width": True, + }, + }, + "v0": { + "__init__": { + "estimator": positive_predictor, + "conformity_score": GammaConformityScore(), + "agg_function": "mean", + "cv": Subsample( + n_resamplings=20, + replace=True, + random_state=RANDOM_STATE + ), + "method": "plus", + "random_state": RANDOM_STATE, + }, + "predict": { + "alpha": [0.5, 0.5], + "optimize_beta": True, + "ensemble": True, + }, + }, + }, + { + "v1": { + "class": JackknifeAfterBootstrapRegressor, + "__init__": { + "confidence_level": 0.9, + "resampling": Subsample( + n_resamplings=30, random_state=RANDOM_STATE + ), + "method": "minmax", + "aggregation_method": "mean", + "random_state": RANDOM_STATE, + }, + "predict_interval": { + "allow_infinite_bounds": True, + }, + }, + "v0": { + "__init__": { + "cv": Subsample(n_resamplings=30, random_state=RANDOM_STATE), + "method": "minmax", + "agg_function": "mean", + "random_state": RANDOM_STATE, + }, + "predict": { + "alpha": 0.1, + "ensemble": True, + "allow_infinite_bounds": True, + }, + }, + }, +] + + +def run_v0_pipeline_cross_or_jackknife(params): + params_ = params["v0"] + mapie_regressor = _MapieRegressor(**params_.get("__init__", {})) + + mapie_regressor.fit(X, y, **params_.get("fit", {})) + preds, pred_intervals = mapie_regressor.predict(X, **params_.get("predict", {})) + + return preds, pred_intervals + + +def run_v1_pipeline_cross_or_jackknife(params): + params_ = params["v1"] + init_params = params_.get("__init__", {}) + confidence_level = init_params.get("confidence_level", 0.9) + confidence_level_length = 1 if isinstance(confidence_level, float) else len( + confidence_level + ) + minimize_interval_width = params_.get("predict_interval", {}).get( + "minimize_interval_width" + ) + + mapie_regressor = params_["class"](**init_params) + mapie_regressor.fit_conformalize(X, y, **params_.get("fit_conformalize", {})) + + X_test = X + preds, pred_intervals = mapie_regressor.predict_interval( + X_test, + **params_.get("predict_interval", {}) + ) + preds_using_predict = mapie_regressor.predict( + X_test, + **params_.get("predict", {}) + ) + + return ( + preds, + pred_intervals, + preds_using_predict, + len(X_test), + confidence_level_length, + minimize_interval_width, + ) + + +@pytest.mark.parametrize( + "params", + params_test_cases_cross + params_test_cases_jackknife +) +def test_cross_and_jackknife_regression(params: dict) -> None: + v0_preds, v0_pred_intervals = run_v0_pipeline_cross_or_jackknife(params) + ( + v1_preds, + v1_pred_intervals, + v1_preds_using_predict, + X_test_length, + confidence_level_length, + minimize_interval_width, + ) = run_v1_pipeline_cross_or_jackknife(params) + + np.testing.assert_array_equal(v0_preds, v1_preds) + np.testing.assert_array_equal(v0_pred_intervals, v1_pred_intervals) + np.testing.assert_array_equal(v1_preds_using_predict, v1_preds) + + if not minimize_interval_width: + # condition to remove when optimize_beta/minimize_interval_width works + # but keep assertion to check shapes + assert v1_pred_intervals.shape == (X_test_length, 2, confidence_level_length) + + +# Below are SplitConformalRegressor and ConformalizedQuantileRegressor tests +# They're using an outdated structure, prefer the style used for CrossConformalRegressor +# and JackknifeAfterBootstrapRegressor above + +params_test_cases_split = [ + { + "v0": { + "alpha": 0.2, + "conformity_score": AbsoluteConformityScore(), + "cv": "split", + "test_size": 0.4, + "sample_weight": sample_weight, + "random_state": RANDOM_STATE, + }, + "v1": { + "confidence_level": 0.8, + "conformity_score": "absolute", + "prefit": False, + "test_size": 0.4, + "fit_params": {"sample_weight": sample_weight_train}, + } + }, + { + "v0": { + "estimator": positive_predictor, + "test_size": 0.2, + "alpha": [0.5, 0.5], + "conformity_score": GammaConformityScore(), + "cv": "split", + "random_state": RANDOM_STATE, + }, + "v1": { + "estimator": positive_predictor, + "test_size": 0.2, + "confidence_level": [0.5, 0.5], + "conformity_score": "gamma", + "prefit": False, + } + }, + { + "v0": { + "estimator": LinearRegression(), + "alpha": 0.1, + "test_size": 0.2, + "conformity_score": ResidualNormalisedScore( + random_state=RANDOM_STATE + ), + "cv": "prefit", + "allow_infinite_bounds": True, + "random_state": RANDOM_STATE, + }, + "v1": { + "estimator": LinearRegression(), + "confidence_level": 0.9, + "prefit": True, + "test_size": 0.2, + "conformity_score": ResidualNormalisedScore( + random_state=RANDOM_STATE + ), + "allow_infinite_bounds": True, + } + }, + { + "v0": { + "estimator": positive_predictor, + "alpha": 0.1, + "conformity_score": GammaConformityScore(), + "cv": "split", + "random_state": RANDOM_STATE, + "test_size": 0.3, + "optimize_beta": True + }, + "v1": { + "estimator": positive_predictor, + "confidence_level": 0.9, + "prefit": False, + "conformity_score": GammaConformityScore(), + "test_size": 0.3, + "minimize_interval_width": True + } + }, +] + + +@pytest.mark.parametrize("params_split", params_test_cases_split) +def test_intervals_and_predictions_exact_equality_split_regression( + params_split: dict) -> None: + v0_params = params_split["v0"] + v1_params = params_split["v1"] + + test_size = v1_params.get("test_size", None) + prefit = v1_params.get("prefit", False) + + compare_model_predictions_and_intervals( + model_v0=_MapieRegressor, + model_v1=SplitConformalRegressor, + X=X, + y=y, + v0_params=v0_params, + v1_params=v1_params, + test_size=test_size, + prefit=prefit, + random_state=RANDOM_STATE, + ) + + +split_model = QuantileRegressor( + solver="highs-ds", + alpha=0.0, + ) + +gbr_models = [] +gbr_alpha = 0.1 + +for alpha_ in [gbr_alpha / 2, (1 - (gbr_alpha / 2)), 0.5]: + estimator_ = GradientBoostingRegressor( + loss='quantile', + alpha=alpha_, + n_estimators=100, + learning_rate=0.1, + max_depth=3 + ) + gbr_models.append(estimator_) + +params_test_cases_quantile = [ + { + "v0": { + "alpha": 0.2, + "cv": "split", + "method": "quantile", + "calib_size": 0.4, + "sample_weight": sample_weight, + "random_state": RANDOM_STATE, + "symmetry": False, + }, + "v1": { + "confidence_level": 0.8, + "prefit": False, + "test_size": 0.4, + "fit_params": {"sample_weight": sample_weight_train}, + }, + }, + { + "v0": { + "estimator": gbr_models, + "alpha": gbr_alpha, + "cv": "prefit", + "method": "quantile", + "calib_size": 0.2, + "sample_weight": sample_weight, + "optimize_beta": True, + "random_state": RANDOM_STATE, + }, + "v1": { + "estimator": gbr_models, + "confidence_level": 1-gbr_alpha, + "prefit": True, + "test_size": 0.2, + "fit_params": {"sample_weight": sample_weight}, + "minimize_interval_width": True, + "symmetric_correction": True, + }, + }, + { + "v0": { + "estimator": split_model, + "alpha": 0.5, + "cv": "split", + "method": "quantile", + "calib_size": 0.3, + "allow_infinite_bounds": True, + "random_state": RANDOM_STATE, + "symmetry": False, + }, + "v1": { + "estimator": split_model, + "confidence_level": 0.5, + "prefit": False, + "test_size": 0.3, + "allow_infinite_bounds": True, + }, + }, + { + "v0": { + "alpha": 0.1, + "cv": "split", + "method": "quantile", + "calib_size": 0.3, + "random_state": RANDOM_STATE, + }, + "v1": { + "confidence_level": 0.9, + "prefit": False, + "test_size": 0.3, + "symmetric_correction": True, + }, + }, +] + + +@pytest.mark.parametrize("params_quantile", params_test_cases_quantile) +def test_intervals_and_predictions_exact_equality_quantile_regression( + params_quantile: dict +) -> None: + v0_params = params_quantile["v0"] + v1_params = params_quantile["v1"] + + test_size = v1_params.get("test_size", None) + prefit = v1_params.get("prefit", False) + + compare_model_predictions_and_intervals( + model_v0=_MapieQuantileRegressor, + model_v1=ConformalizedQuantileRegressor, + X=X, + y=y, + v0_params=v0_params, + v1_params=v1_params, + test_size=test_size, + prefit=prefit, + random_state=RANDOM_STATE, + ) + + +def compare_model_predictions_and_intervals( + model_v0: Type[_MapieRegressor], + model_v1: Type[Union[ + SplitConformalRegressor, + CrossConformalRegressor, + JackknifeAfterBootstrapRegressor, + ConformalizedQuantileRegressor + ]], + X: NDArray, + y: NDArray, + v0_params: Dict = {}, + v1_params: Dict = {}, + prefit: bool = False, + test_size: Optional[float] = None, + random_state: int = RANDOM_STATE, +) -> None: + if v0_params.get("alpha"): + if isinstance(v0_params["alpha"], float): + n_alpha = 1 + else: + n_alpha = len(v0_params["alpha"]) + else: + n_alpha = 1 + + if test_size is not None: + X_train, X_conf, y_train, y_conf = train_test_split_shuffle( + X, + y, + test_size=test_size, + random_state=random_state, + ) + else: + X_train, X_conf, y_train, y_conf = X, X, y, y + + if prefit: + estimator = v0_params["estimator"] + if isinstance(estimator, list): + for single_estimator in estimator: + single_estimator.fit(X_train, y_train) + else: + estimator.fit(X_train, y_train) + + v0_params["estimator"] = estimator + v1_params["estimator"] = estimator + + v0_init_params = filter_params(model_v0.__init__, v0_params) + v1_init_params = filter_params(model_v1.__init__, v1_params) + + v0 = model_v0(**v0_init_params) + v1 = model_v1(**v1_init_params) + + v0_fit_params = filter_params(v0.fit, v0_params) + v1_fit_params = filter_params(v1.fit, v1_params) + v1_conformalize_params = filter_params(v1.conformalize, v1_params) + + if prefit: + v0.fit(X_conf, y_conf, **v0_fit_params) + else: + v0.fit(X, y, **v0_fit_params) + v1.fit(X_train, y_train, **v1_fit_params) + + v1.conformalize(X_conf, y_conf, **v1_conformalize_params) + + v0_predict_params = filter_params(v0.predict, v0_params) + if 'alpha' in v0_init_params: + v0_predict_params.pop('alpha') + + v1_predict_params = filter_params(v1.predict, v1_params) + v1_predict_interval_params = filter_params(v1.predict_interval, v1_params) + + v0_preds, v0_pred_intervals = v0.predict(X_conf, **v0_predict_params) + v1_preds, v1_pred_intervals = v1.predict_interval( + X_conf, **v1_predict_interval_params + ) + + v1_preds_using_predict: ArrayLike = v1.predict(X_conf, **v1_predict_params) + + np.testing.assert_array_equal(v0_preds, v1_preds) + np.testing.assert_array_equal(v0_pred_intervals, v1_pred_intervals) + np.testing.assert_array_equal(v1_preds_using_predict, v1_preds) + if not v0_params.get("optimize_beta"): + # condition to remove when optimize_beta works + # keep assertion + assert v1_pred_intervals.shape == (len(X_conf), 2, n_alpha) diff --git a/mapie/tests/test_regression.py b/mapie/tests/test_regression.py index 4dbe52e4a..379fdea35 100644 --- a/mapie/tests/test_regression.py +++ b/mapie/tests/test_regression.py @@ -13,7 +13,7 @@ from sklearn.dummy import DummyRegressor from sklearn.ensemble import GradientBoostingRegressor from sklearn.impute import SimpleImputer -from sklearn.linear_model import LinearRegression +from sklearn.linear_model import LinearRegression, QuantileRegressor from sklearn.model_selection import ( GroupKFold, KFold, LeaveOneOut, PredefinedSplit, ShuffleSplit, train_test_split, LeaveOneGroupOut, LeavePGroupsOut @@ -33,7 +33,9 @@ from mapie.metrics.regression import ( regression_coverage_score, ) -from mapie.regression.regression import _MapieRegressor +from mapie.regression import ConformalizedQuantileRegressor +from mapie.regression.regression import _MapieRegressor, \ + JackknifeAfterBootstrapRegressor, SplitConformalRegressor, CrossConformalRegressor from mapie.subsample import Subsample X_toy = np.array([0, 1, 2, 3, 4, 5]).reshape(-1, 1) @@ -1046,3 +1048,200 @@ def test_invalid_method(method: str) -> None: ValueError, match="(Invalid method.)|(Invalid conformity score.)*" ): mapie_estimator.fit(X_toy, y_toy) + + +class TestCheckAndConvertResamplingToCv: + def test_with_integer(self): + regressor = JackknifeAfterBootstrapRegressor() + cv = regressor._check_and_convert_resampling_to_cv(50) + + assert isinstance(cv, Subsample) + assert cv.n_resamplings == 50 + + def test_with_subsample(self): + custom_subsample = Subsample(n_resamplings=25, random_state=42) + regressor = JackknifeAfterBootstrapRegressor() + cv = regressor._check_and_convert_resampling_to_cv(custom_subsample) + + assert cv is custom_subsample + + def test_with_invalid_input(self): + regressor = JackknifeAfterBootstrapRegressor() + + with pytest.raises( + ValueError, + match="resampling must be an integer or a Subsample instance" + ): + regressor._check_and_convert_resampling_to_cv("invalid_input") + + +@pytest.fixture(scope="module") +def dataset_regression(): + X, y = make_regression( + n_samples=500, n_features=2, noise=1.0, random_state=random_state + ) + X_train, X_conf_test, y_train, y_conf_test = train_test_split( + X, y, random_state=random_state + ) + X_conformalize, X_test, y_conformalize, y_test = train_test_split( + X_conf_test, y_conf_test, random_state=random_state + ) + return X_train, X_conformalize, X_test, y_train, y_conformalize, y_test + + +def test_scr_same_predictions_prefit_not_prefit(dataset_regression) -> None: + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = ( + dataset_regression) + regressor = LinearRegression() + regressor.fit(X_train, y_train) + scr_prefit = SplitConformalRegressor(estimator=regressor, prefit=True) + scr_prefit.conformalize(X_conformalize, y_conformalize) + predictions_scr_prefit = scr_prefit.predict_interval(X_test) + + scr_not_prefit = SplitConformalRegressor(estimator=LinearRegression(), prefit=False) + scr_not_prefit.fit(X_train, y_train).conformalize(X_conformalize, y_conformalize) + predictions_scr_not_prefit = scr_not_prefit.predict_interval(X_test) + np.testing.assert_equal(predictions_scr_prefit, predictions_scr_not_prefit) + + +@pytest.mark.parametrize( + "split_technique,predict_method,dataset,estimator_class", + [ + ( + SplitConformalRegressor, + "predict_interval", + "dataset_regression", + DummyRegressor + ), + ( + ConformalizedQuantileRegressor, + "predict_interval", + "dataset_regression", + QuantileRegressor + ) + ] +) +class TestWrongMethodsOrderRaisesErrorForSplitTechniquesRegression: + def test_with_prefit_false( + self, + split_technique, + predict_method, + dataset, + estimator_class, + request + ): + dataset = request.getfixturevalue(dataset) + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset + estimator = estimator_class() + technique = split_technique(estimator=estimator, prefit=False) + + with pytest.raises(ValueError, match=r"call fit before calling conformalize"): + technique.conformalize( + X_conformalize, + y_conformalize + ) + + technique.fit(X_train, y_train) + + with pytest.raises(ValueError, match=r"fit method already called"): + technique.fit(X_train, y_train) + with pytest.raises( + ValueError, + match=r"call conformalize before calling predict" + ): + technique.predict(X_test) + + with pytest.raises( + ValueError, + match=f"call conformalize before calling {predict_method}" + ): + getattr(technique, predict_method)(X_test) + + technique.conformalize(X_conformalize, y_conformalize) + + with pytest.raises(ValueError, match=r"conformalize method already called"): + technique.conformalize(X_conformalize, y_conformalize) + + def test_with_prefit_true( + self, + split_technique, + predict_method, + dataset, + estimator_class, + request + ): + dataset = request.getfixturevalue(dataset) + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset + estimator = estimator_class() + estimator.fit(X_train, y_train) + + if split_technique == ConformalizedQuantileRegressor: + technique = split_technique(estimator=[estimator] * 3, prefit=True) + else: + technique = split_technique(estimator=estimator, prefit=True) + + with pytest.raises(ValueError, match=r"The fit method must be skipped"): + technique.fit(X_train, y_train) + with pytest.raises( + ValueError, + match=r"call conformalize before calling predict" + ): + technique.predict(X_test) + + with pytest.raises( + ValueError, + match=f"call conformalize before calling {predict_method}" + ): + getattr(technique, predict_method)(X_test) + + technique.conformalize(X_conformalize, y_conformalize) + + with pytest.raises(ValueError, match=r"conformalize method already called"): + technique.conformalize(X_conformalize, y_conformalize) + + +@pytest.mark.parametrize( + "cross_technique,predict_method,dataset,estimator_class", + [ + ( + CrossConformalRegressor, + "predict_interval", + "dataset_regression", + DummyRegressor + ), + ( + JackknifeAfterBootstrapRegressor, + "predict_interval", + "dataset_regression", + DummyRegressor + ) + ] +) +class TestWrongMethodsOrderRaisesErrorForCrossTechniques: + def test_wrong_methods_order( + self, + cross_technique, + predict_method, + dataset, + estimator_class, + request + ): + dataset = request.getfixturevalue(dataset) + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset + technique = cross_technique(estimator=estimator_class()) + + with pytest.raises( + ValueError, + match=r"call fit_conformalize before calling predict" + ): + technique.predict(X_test) + with pytest.raises( + ValueError, + match=f"call fit_conformalize before calling {predict_method}" + ): + getattr(technique, predict_method)(X_test) + + technique.fit_conformalize(X_conformalize, y_conformalize) + + with pytest.raises(ValueError, match=r"fit_conformalize method already called"): + technique.fit_conformalize(X_conformalize, y_conformalize) diff --git a/mapie/tests/test_utils.py b/mapie/tests/test_utils.py index cb1da4161..5a1d41f44 100644 --- a/mapie/tests/test_utils.py +++ b/mapie/tests/test_utils.py @@ -1,12 +1,15 @@ from __future__ import annotations +import inspect import logging import re -from typing import Any, Optional, Tuple +from typing import Any, Optional, Tuple, Union, Callable, Dict +from unittest.mock import patch import numpy as np import pytest from numpy.random import RandomState +from sklearn.base import BaseEstimator, ClassifierMixin from sklearn.datasets import make_regression from sklearn.linear_model import LinearRegression from sklearn.model_selection import (BaseCrossValidator, KFold, LeaveOneOut, @@ -14,6 +17,8 @@ from sklearn.utils.validation import check_is_fitted from numpy.typing import ArrayLike, NDArray +from typing_extensions import Self + from mapie.regression.quantile_regression import _MapieQuantileRegressor from mapie.utils import (_check_alpha, _check_alpha_and_n_samples, _check_array_inf, _check_array_nan, _check_arrays_length, @@ -22,7 +27,17 @@ _check_n_jobs, _check_n_samples, _check_no_agg_cv, _check_null_weight, _check_number_bins, _check_split_strategy, _check_verbose, - _compute_quantiles, _fit_estimator, _get_binning_groups) + _compute_quantiles, _fit_estimator, _get_binning_groups, + train_conformalize_test_split, + _transform_confidence_level_to_alpha, + _transform_confidence_level_to_alpha_list, + _check_if_param_in_allowed_values, _check_cv_not_string, + _cast_point_predictions_to_ndarray, + _cast_predictions_to_ndarray_tuple, _prepare_params, + _prepare_fit_params_and_sample_weight, + _raise_error_if_previous_method_not_called, + _raise_error_if_method_already_called, + _raise_error_if_fit_called_in_prefit_mode) X_toy = np.array([0, 1, 2, 3, 4, 5]).reshape(-1, 1) y_toy = np.array([5, 7, 9, 11, 13, 15]) @@ -564,3 +579,315 @@ def test_invalid_n_samples_float(n_samples: float) -> None: ) ): _check_n_samples(X=X, n_samples=n_samples, indices=indices) + + +def train_test_split_shuffle( + X: NDArray, + y: NDArray, + test_size: float = None, + random_state: int = 42, + sample_weight: Optional[NDArray] = None, +) -> Union[Tuple[Any, Any, Any, Any], Tuple[Any, Any, Any, Any, Any, Any]]: + splitter = ShuffleSplit( + n_splits=1, + test_size=test_size, + random_state=random_state + ) + train_idx, test_idx = next(splitter.split(X)) + + X_train, X_test = X[train_idx], X[test_idx] + y_train, y_test = y[train_idx], y[test_idx] + if sample_weight is not None: + sample_weight_train = sample_weight[train_idx] + sample_weight_test = sample_weight[test_idx] + return X_train, X_test, y_train, y_test, sample_weight_train, sample_weight_test + + return X_train, X_test, y_train, y_test + + +def filter_params( + function: Callable, + params: Optional[Dict[str, Any]] = None +) -> Dict[str, Any]: + if params is None: + return {} + + model_params = inspect.signature(function).parameters + return {k: v for k, v in params.items() if k in model_params} + + +class DummyClassifierWithFitAndPredictParams(BaseEstimator, ClassifierMixin): + def __init__(self): + self.classes_ = None + self._dummy_fit_param = None + + def fit(self, X: ArrayLike, y: ArrayLike, dummy_fit_param: bool = False) -> Self: + self.classes_ = np.unique(y) + if len(self.classes_) < 2: + raise ValueError("Dummy classifier needs at least 3 classes") + self._dummy_fit_param = dummy_fit_param + return self + + def predict_proba(self, X: ArrayLike, dummy_predict_param: bool = False) -> NDArray: + probas = np.zeros((len(X), len(self.classes_))) + if self._dummy_fit_param & dummy_predict_param: + probas[:, 0] = 0.1 + probas[:, 1] = 0.9 + elif self._dummy_fit_param: + probas[:, 1] = 0.1 + probas[:, 2] = 0.9 + elif dummy_predict_param: + probas[:, 1] = 0.1 + probas[:, 0] = 0.9 + else: + probas[:, 2] = 0.1 + probas[:, 0] = 0.9 + return probas + + def predict(self, X: ArrayLike, dummy_predict_param: bool = False) -> NDArray: + y_preds_proba = self.predict_proba(X, dummy_predict_param) + return np.amax(y_preds_proba, axis=0) + + +@pytest.fixture(scope="module") +def dataset(): + X, y = make_regression( + n_samples=100, n_features=2, noise=1.0, random_state=random_state + ) + return X, y + + +class TestTrainConformalizeTestSplit: + + def test_error_sum_int_is_not_dataset_size(self, dataset): + X, y = dataset + with pytest.raises(ValueError): + train_conformalize_test_split( + X, y, train_size=1, conformalize_size=1, + test_size=1, random_state=random_state + ) + + def test_error_sum_float_is_not_1(self, dataset): + X, y = dataset + with pytest.raises(ValueError): + train_conformalize_test_split( + X, y, train_size=0.5, conformalize_size=0.5, + test_size=0.5, random_state=random_state + ) + + def test_error_sizes_are_int_and_float(self, dataset): + X, y = dataset + with pytest.raises(TypeError): + train_conformalize_test_split( + X, y, train_size=5, conformalize_size=0.5, + test_size=0.5, random_state=random_state + ) + + def test_3_floats(self, dataset): + X, y = dataset + ( + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test + ) = train_conformalize_test_split( + X, y, train_size=0.6, conformalize_size=0.2, + test_size=0.2, random_state=random_state + ) + assert len(X_train) == 60 + assert len(X_conformalize) == 20 + assert len(X_test) == 20 + + def test_3_ints(self, dataset): + X, y = dataset + ( + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test + ) = train_conformalize_test_split( + X, y, train_size=60, conformalize_size=20, + test_size=20, random_state=random_state + ) + assert len(X_train) == 60 + assert len(X_conformalize) == 20 + assert len(X_test) == 20 + + def test_random_state(self, dataset): + X, y = dataset + ( + X_train_1, X_conformalize_1, X_test_1, y_train_1, y_conformalize_1, y_test_1 + ) = train_conformalize_test_split( + X, y, train_size=60, conformalize_size=20, + test_size=20, random_state=random_state + ) + ( + X_train_2, X_conformalize_2, X_test_2, y_train_2, y_conformalize_2, y_test_2 + ) = train_conformalize_test_split( + X, y, train_size=60, conformalize_size=20, + test_size=20, random_state=random_state + ) + assert np.array_equal(X_train_1, X_train_2) + assert np.array_equal(X_conformalize_1, X_conformalize_2) + assert np.array_equal(X_test_1, X_test_2) + assert np.array_equal(y_train_1, y_train_2) + assert np.array_equal(y_conformalize_1, y_conformalize_2) + assert np.array_equal(y_test_1, y_test_2) + + def test_different_random_state(self, dataset): + X, y = dataset + ( + X_train_1, X_conformalize_1, X_test_1, y_train_1, y_conformalize_1, y_test_1 + ) = train_conformalize_test_split( + X, y, train_size=60, conformalize_size=20, + test_size=20, random_state=random_state + ) + ( + X_train_2, X_conformalize_2, X_test_2, y_train_2, y_conformalize_2, y_test_2 + ) = train_conformalize_test_split( + X, y, train_size=60, conformalize_size=20, + test_size=20, random_state=random_state + 1 + ) + assert not np.array_equal(X_train_1, X_train_2) + assert not np.array_equal(X_conformalize_1, X_conformalize_2) + assert not np.array_equal(X_test_1, X_test_2) + assert not np.array_equal(y_train_1, y_train_2) + assert not np.array_equal(y_conformalize_1, y_conformalize_2) + assert not np.array_equal(y_test_1, y_test_2) + + def test_shuffle_false(self, dataset): + X, y = dataset + ( + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test + ) = train_conformalize_test_split( + X, y, train_size=60, conformalize_size=20, + test_size=20, random_state=random_state, shuffle=False + ) + assert np.array_equal(np.concatenate((y_train, y_conformalize, y_test)), y) + + +@pytest.fixture +def point_predictions(): + return np.array([1, 2, 3]) + + +@pytest.fixture +def point_and_interval_predictions(): + return np.array([1, 2]), np.array([3, 4]) + + +@pytest.mark.parametrize( + "confidence_level, expected", + [ + (0.9, 0.1), + (0.7, 0.3), + (0.999, 0.001), + ] +) +def test_transform_confidence_level_to_alpha(confidence_level, expected): + result = _transform_confidence_level_to_alpha(confidence_level) + assert result == expected + assert str(result) == str(expected) # Ensure clean representation + + +class TestTransformConfidenceLevelToAlphaList: + def test_non_list_iterable(self): + confidence_level = (0.8, 0.7) # Testing a non-list iterable + assert _transform_confidence_level_to_alpha_list(confidence_level) == [0.2, 0.3] + + def test_transform_confidence_level_to_alpha_is_called(self): + with patch( + 'mapie.utils._transform_confidence_level_to_alpha' + ) as mock_transform_confidence_level_to_alpha: + _transform_confidence_level_to_alpha_list([0.2, 0.3]) + mock_transform_confidence_level_to_alpha.assert_called() + + +class TestCheckIfParamInAllowedValues: + def test_error(self): + with pytest.raises(ValueError): + _check_if_param_in_allowed_values("invalid_option", "", ["valid_option"]) + + def test_ok(self): + assert _check_if_param_in_allowed_values("valid", "", ["valid"]) is None + + +def test_check_cv_not_string(): + with pytest.raises(ValueError): + _check_cv_not_string("string") + + +class TestCastPointPredictionsToNdarray: + def test_error(self, point_and_interval_predictions): + with pytest.raises(TypeError): + _cast_point_predictions_to_ndarray(point_and_interval_predictions) + + def test_valid_ndarray(self, point_predictions): + point_predictions = np.array([1, 2, 3]) + result = _cast_point_predictions_to_ndarray(point_predictions) + assert result is point_predictions + assert isinstance(result, np.ndarray) + + +class TestCastPredictionsToNdarrayTuple: + def test_error(self, point_predictions): + with pytest.raises(TypeError): + _cast_predictions_to_ndarray_tuple(point_predictions) + + def test_valid_ndarray(self, point_and_interval_predictions): + result = _cast_predictions_to_ndarray_tuple(point_and_interval_predictions) + assert result is point_and_interval_predictions + assert isinstance(result, tuple) + assert isinstance(result[0], np.ndarray) + assert isinstance(result[1], np.ndarray) + + +@pytest.mark.parametrize( + "params, expected", [(None, {}), ({"a": 1, "b": 2}, {"a": 1, "b": 2})] +) +def test_prepare_params(params, expected): + assert _prepare_params(params) == expected + assert _prepare_params(params) is not params + + +class TestPrepareFitParamsAndSampleWeight: + def test_uses_prepare_params(self): + with patch('mapie.utils._prepare_params') as mock_prepare_params: + _prepare_fit_params_and_sample_weight({"param1": 1}) + mock_prepare_params.assert_called() + + def test_with_sample_weight(self): + fit_params = {"sample_weight": [0.1, 0.2, 0.3]} + assert _prepare_fit_params_and_sample_weight(fit_params) == ( + {}, + [0.1, 0.2, 0.3] + ) + + def test_without_sample_weight(self): + params = {"param1": 1} + assert _prepare_fit_params_and_sample_weight(params) == (params, None) + + +class TestRaiseErrorIfPreviousMethodNotCalled: + def test_raises_error_when_previous_method_not_called(self): + with pytest.raises(ValueError): + _raise_error_if_previous_method_not_called( + "current_method", "previous_method", False + ) + + def test_does_nothing_when_previous_method_called(self): + assert _raise_error_if_previous_method_not_called( + "current_method", "previous_method", True + ) is None + + +class TestRaiseErrorIfMethodAlreadyCalled: + def test_raises_error_when_method_already_called(self): + with pytest.raises(ValueError): + _raise_error_if_method_already_called("method", True) + + def test_does_nothing_when_method_not_called(self): + assert _raise_error_if_method_already_called("method", False) is None + + +class TestRaiseErrorIfFitCalledInPrefitMode: + def test_raises_error_in_prefit_mode(self): + with pytest.raises(ValueError): + _raise_error_if_fit_called_in_prefit_mode(True) + + def test_does_nothing_when_not_in_prefit_mode(self): + assert _raise_error_if_fit_called_in_prefit_mode(False) is None diff --git a/tests_v1/test_functional/test_functional.py b/tests_v1/test_functional/test_functional.py deleted file mode 100644 index 56daf3681..000000000 --- a/tests_v1/test_functional/test_functional.py +++ /dev/null @@ -1,212 +0,0 @@ -import numpy as np -import pytest -from sklearn.datasets import make_regression, make_classification -from sklearn.linear_model import LinearRegression, QuantileRegressor -from sklearn.dummy import DummyRegressor, DummyClassifier -from sklearn.model_selection import train_test_split -from mapie.regression import ( - SplitConformalRegressor, - CrossConformalRegressor, - ConformalizedQuantileRegressor, JackknifeAfterBootstrapRegressor, -) -from mapie.classification import SplitConformalClassifier, CrossConformalClassifier - -RANDOM_STATE = 1 - - -@pytest.fixture(scope="module") -def dataset_regression(): - X, y = make_regression( - n_samples=500, n_features=2, noise=1.0, random_state=RANDOM_STATE - ) - X_train, X_conf_test, y_train, y_conf_test = train_test_split( - X, y, random_state=RANDOM_STATE - ) - X_conformalize, X_test, y_conformalize, y_test = train_test_split( - X_conf_test, y_conf_test, random_state=RANDOM_STATE - ) - return X_train, X_conformalize, X_test, y_train, y_conformalize, y_test - - -@pytest.fixture(scope="module") -def dataset_classification(): - X, y = make_classification( - n_samples=500, n_informative=5, n_classes=4, random_state=RANDOM_STATE, - ) - X_train, X_conf_test, y_train, y_conf_test = train_test_split( - X, y, random_state=RANDOM_STATE - ) - X_conformalize, X_test, y_conformalize, y_test = train_test_split( - X_conf_test, y_conf_test, random_state=RANDOM_STATE - ) - return X_train, X_conformalize, X_test, y_train, y_conformalize, y_test - - -def test_scr_same_predictions_prefit_not_prefit(dataset_regression) -> None: - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = ( - dataset_regression) - regressor = LinearRegression() - regressor.fit(X_train, y_train) - scr_prefit = SplitConformalRegressor(estimator=regressor, prefit=True) - scr_prefit.conformalize(X_conformalize, y_conformalize) - predictions_scr_prefit = scr_prefit.predict_interval(X_test) - - scr_not_prefit = SplitConformalRegressor(estimator=LinearRegression(), prefit=False) - scr_not_prefit.fit(X_train, y_train).conformalize(X_conformalize, y_conformalize) - predictions_scr_not_prefit = scr_not_prefit.predict_interval(X_test) - np.testing.assert_equal(predictions_scr_prefit, predictions_scr_not_prefit) - - -@pytest.mark.parametrize( - "split_technique,predict_method,dataset,estimator_class", - [ - ( - SplitConformalRegressor, - "predict_interval", - "dataset_regression", - DummyRegressor - ), - ( - ConformalizedQuantileRegressor, - "predict_interval", - "dataset_regression", - QuantileRegressor - ), - ( - SplitConformalClassifier, - "predict_set", - "dataset_classification", - DummyClassifier - ) - ] -) -class TestWrongMethodsOrderRaisesErrorForSplitTechniques: - def test_with_prefit_false( - self, - split_technique, - predict_method, - dataset, - estimator_class, - request - ): - dataset = request.getfixturevalue(dataset) - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset - estimator = estimator_class() - technique = split_technique(estimator=estimator, prefit=False) - - with pytest.raises(ValueError, match=r"call fit before calling conformalize"): - technique.conformalize( - X_conformalize, - y_conformalize - ) - - technique.fit(X_train, y_train) - - with pytest.raises(ValueError, match=r"fit method already called"): - technique.fit(X_train, y_train) - with pytest.raises( - ValueError, - match=r"call conformalize before calling predict" - ): - technique.predict(X_test) - - with pytest.raises( - ValueError, - match=f"call conformalize before calling {predict_method}" - ): - getattr(technique, predict_method)(X_test) - - technique.conformalize(X_conformalize, y_conformalize) - - with pytest.raises(ValueError, match=r"conformalize method already called"): - technique.conformalize(X_conformalize, y_conformalize) - - def test_with_prefit_true( - self, - split_technique, - predict_method, - dataset, - estimator_class, - request - ): - dataset = request.getfixturevalue(dataset) - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset - estimator = estimator_class() - estimator.fit(X_train, y_train) - - if split_technique == ConformalizedQuantileRegressor: - technique = split_technique(estimator=[estimator] * 3, prefit=True) - else: - technique = split_technique(estimator=estimator, prefit=True) - - with pytest.raises(ValueError, match=r"The fit method must be skipped"): - technique.fit(X_train, y_train) - with pytest.raises( - ValueError, - match=r"call conformalize before calling predict" - ): - technique.predict(X_test) - - with pytest.raises( - ValueError, - match=f"call conformalize before calling {predict_method}" - ): - getattr(technique, predict_method)(X_test) - - technique.conformalize(X_conformalize, y_conformalize) - - with pytest.raises(ValueError, match=r"conformalize method already called"): - technique.conformalize(X_conformalize, y_conformalize) - - -@pytest.mark.parametrize( - "cross_technique,predict_method,dataset,estimator_class", - [ - ( - CrossConformalRegressor, - "predict_interval", - "dataset_regression", - DummyRegressor - ), - ( - JackknifeAfterBootstrapRegressor, - "predict_interval", - "dataset_regression", - DummyRegressor - ), - ( - CrossConformalClassifier, - "predict_set", - "dataset_classification", - DummyClassifier - ), - ] -) -class TestWrongMethodsOrderRaisesErrorForCrossTechniques: - def test_wrong_methods_order( - self, - cross_technique, - predict_method, - dataset, - estimator_class, - request - ): - dataset = request.getfixturevalue(dataset) - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset - technique = cross_technique(estimator=estimator_class()) - - with pytest.raises( - ValueError, - match=r"call fit_conformalize before calling predict" - ): - technique.predict(X_test) - with pytest.raises( - ValueError, - match=f"call fit_conformalize before calling {predict_method}" - ): - getattr(technique, predict_method)(X_test) - - technique.fit_conformalize(X_conformalize, y_conformalize) - - with pytest.raises(ValueError, match=r"fit_conformalize method already called"): - technique.fit_conformalize(X_conformalize, y_conformalize) diff --git a/tests_v1/test_functional/test_non_regression_classification.py b/tests_v1/test_functional/test_non_regression_classification.py index 58be6b6f1..73391ddd8 100644 --- a/tests_v1/test_functional/test_non_regression_classification.py +++ b/tests_v1/test_functional/test_non_regression_classification.py @@ -17,7 +17,7 @@ TopKConformityScore, LACConformityScore, ) -from tests_v1.test_functional.utils import ( +from mapie.tests.test_utils import ( DummyClassifierWithFitAndPredictParams, train_test_split_shuffle, ) diff --git a/tests_v1/test_functional/test_non_regression_regression.py b/tests_v1/test_functional/test_non_regression_regression.py index 2d25975b4..a9dec18f6 100644 --- a/tests_v1/test_functional/test_non_regression_regression.py +++ b/tests_v1/test_functional/test_non_regression_regression.py @@ -22,7 +22,7 @@ from mapie.regression.regression import _MapieRegressor from mapie.regression.quantile_regression import _MapieQuantileRegressor -from tests_v1.test_functional.utils import filter_params, train_test_split_shuffle +from mapie.tests.test_utils import filter_params, train_test_split_shuffle from sklearn.model_selection import LeaveOneOut, GroupKFold RANDOM_STATE = 1 diff --git a/tests_v1/test_functional/utils.py b/tests_v1/test_functional/utils.py deleted file mode 100644 index 576f3054d..000000000 --- a/tests_v1/test_functional/utils.py +++ /dev/null @@ -1,77 +0,0 @@ -from typing import Callable, Dict, Any, Optional, Tuple, Union - -import numpy as np -from sklearn.base import BaseEstimator, ClassifierMixin -from typing_extensions import Self - -from numpy.typing import NDArray, ArrayLike -import inspect -from sklearn.model_selection import ShuffleSplit - - -def train_test_split_shuffle( - X: NDArray, - y: NDArray, - test_size: float = None, - random_state: int = 42, - sample_weight: Optional[NDArray] = None, -) -> Union[Tuple[Any, Any, Any, Any], Tuple[Any, Any, Any, Any, Any, Any]]: - splitter = ShuffleSplit( - n_splits=1, - test_size=test_size, - random_state=random_state - ) - train_idx, test_idx = next(splitter.split(X)) - - X_train, X_test = X[train_idx], X[test_idx] - y_train, y_test = y[train_idx], y[test_idx] - if sample_weight is not None: - sample_weight_train = sample_weight[train_idx] - sample_weight_test = sample_weight[test_idx] - return X_train, X_test, y_train, y_test, sample_weight_train, sample_weight_test - - return X_train, X_test, y_train, y_test - - -def filter_params( - function: Callable, - params: Optional[Dict[str, Any]] = None -) -> Dict[str, Any]: - if params is None: - return {} - - model_params = inspect.signature(function).parameters - return {k: v for k, v in params.items() if k in model_params} - - -class DummyClassifierWithFitAndPredictParams(BaseEstimator, ClassifierMixin): - def __init__(self): - self.classes_ = None - self._dummy_fit_param = None - - def fit(self, X: ArrayLike, y: ArrayLike, dummy_fit_param: bool = False) -> Self: - self.classes_ = np.unique(y) - if len(self.classes_) < 2: - raise ValueError("Dummy classifier needs at least 3 classes") - self._dummy_fit_param = dummy_fit_param - return self - - def predict_proba(self, X: ArrayLike, dummy_predict_param: bool = False) -> NDArray: - probas = np.zeros((len(X), len(self.classes_))) - if self._dummy_fit_param & dummy_predict_param: - probas[:, 0] = 0.1 - probas[:, 1] = 0.9 - elif self._dummy_fit_param: - probas[:, 1] = 0.1 - probas[:, 2] = 0.9 - elif dummy_predict_param: - probas[:, 1] = 0.1 - probas[:, 0] = 0.9 - else: - probas[:, 2] = 0.1 - probas[:, 0] = 0.9 - return probas - - def predict(self, X: ArrayLike, dummy_predict_param: bool = False) -> NDArray: - y_preds_proba = self.predict_proba(X, dummy_predict_param) - return np.amax(y_preds_proba, axis=0) diff --git a/tests_v1/test_unit/test_conformity_scores_utils.py b/tests_v1/test_unit/test_conformity_scores_utils.py deleted file mode 100644 index e27c11e60..000000000 --- a/tests_v1/test_unit/test_conformity_scores_utils.py +++ /dev/null @@ -1,37 +0,0 @@ -import pytest - -from mapie.conformity_scores.utils import ( - check_and_select_conformity_score, -) -from mapie.conformity_scores.regression import BaseRegressionScore -from mapie.conformity_scores.classification import BaseClassificationScore -from mapie.conformity_scores.bounds import ( - AbsoluteConformityScore, - GammaConformityScore, -) -from mapie.conformity_scores.sets import ( - LACConformityScore, - TopKConformityScore, -) - - -class TestCheckAndSelectConformityScore: - - @pytest.mark.parametrize( - "score, score_type, expected_class", [ - (AbsoluteConformityScore(), BaseRegressionScore, AbsoluteConformityScore), - ("gamma", BaseRegressionScore, GammaConformityScore), - (LACConformityScore(), BaseClassificationScore, LACConformityScore), - ("top_k", BaseClassificationScore, TopKConformityScore), - ] - ) - def test_with_valid_inputs(self, score, score_type, expected_class): - result = check_and_select_conformity_score(score, score_type) - assert isinstance(result, expected_class) - - @pytest.mark.parametrize( - "score_type", [BaseRegressionScore, BaseClassificationScore] - ) - def test_with_invalid_input(self, score_type): - with pytest.raises(ValueError): - check_and_select_conformity_score("I'm not a valid input :(", score_type) diff --git a/tests_v1/test_unit/test_regression.py b/tests_v1/test_unit/test_regression.py deleted file mode 100644 index 14a75c627..000000000 --- a/tests_v1/test_unit/test_regression.py +++ /dev/null @@ -1,28 +0,0 @@ -import pytest -from mapie.subsample import Subsample -from mapie.regression import JackknifeAfterBootstrapRegressor - - -class TestCheckAndConvertResamplingToCv: - def test_with_integer(self): - regressor = JackknifeAfterBootstrapRegressor() - cv = regressor._check_and_convert_resampling_to_cv(50) - - assert isinstance(cv, Subsample) - assert cv.n_resamplings == 50 - - def test_with_subsample(self): - custom_subsample = Subsample(n_resamplings=25, random_state=42) - regressor = JackknifeAfterBootstrapRegressor() - cv = regressor._check_and_convert_resampling_to_cv(custom_subsample) - - assert cv is custom_subsample - - def test_with_invalid_input(self): - regressor = JackknifeAfterBootstrapRegressor() - - with pytest.raises( - ValueError, - match="resampling must be an integer or a Subsample instance" - ): - regressor._check_and_convert_resampling_to_cv("invalid_input") diff --git a/tests_v1/test_unit/test_utils.py b/tests_v1/test_unit/test_utils.py deleted file mode 100644 index 4f31615cf..000000000 --- a/tests_v1/test_unit/test_utils.py +++ /dev/null @@ -1,266 +0,0 @@ -import numpy as np -import pytest -from sklearn.datasets import make_regression - -from mapie.utils import ( - _prepare_params, - _prepare_fit_params_and_sample_weight, - _transform_confidence_level_to_alpha_list, - _transform_confidence_level_to_alpha, - _check_if_param_in_allowed_values, - _check_cv_not_string, - _cast_point_predictions_to_ndarray, - _cast_predictions_to_ndarray_tuple, - _raise_error_if_previous_method_not_called, - _raise_error_if_method_already_called, - _raise_error_if_fit_called_in_prefit_mode, - train_conformalize_test_split -) -from unittest.mock import patch - - -RANDOM_STATE = 1 - - -@pytest.fixture(scope="module") -def dataset(): - X, y = make_regression( - n_samples=100, n_features=2, noise=1.0, random_state=RANDOM_STATE - ) - return X, y - - -class TestTrainConformalizeTestSplit: - - def test_error_sum_int_is_not_dataset_size(self, dataset): - X, y = dataset - with pytest.raises(ValueError): - train_conformalize_test_split( - X, y, train_size=1, conformalize_size=1, - test_size=1, random_state=RANDOM_STATE - ) - - def test_error_sum_float_is_not_1(self, dataset): - X, y = dataset - with pytest.raises(ValueError): - train_conformalize_test_split( - X, y, train_size=0.5, conformalize_size=0.5, - test_size=0.5, random_state=RANDOM_STATE - ) - - def test_error_sizes_are_int_and_float(self, dataset): - X, y = dataset - with pytest.raises(TypeError): - train_conformalize_test_split( - X, y, train_size=5, conformalize_size=0.5, - test_size=0.5, random_state=RANDOM_STATE - ) - - def test_3_floats(self, dataset): - X, y = dataset - ( - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test - ) = train_conformalize_test_split( - X, y, train_size=0.6, conformalize_size=0.2, - test_size=0.2, random_state=RANDOM_STATE - ) - assert len(X_train) == 60 - assert len(X_conformalize) == 20 - assert len(X_test) == 20 - - def test_3_ints(self, dataset): - X, y = dataset - ( - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test - ) = train_conformalize_test_split( - X, y, train_size=60, conformalize_size=20, - test_size=20, random_state=RANDOM_STATE - ) - assert len(X_train) == 60 - assert len(X_conformalize) == 20 - assert len(X_test) == 20 - - def test_random_state(self, dataset): - X, y = dataset - ( - X_train_1, X_conformalize_1, X_test_1, y_train_1, y_conformalize_1, y_test_1 - ) = train_conformalize_test_split( - X, y, train_size=60, conformalize_size=20, - test_size=20, random_state=RANDOM_STATE - ) - ( - X_train_2, X_conformalize_2, X_test_2, y_train_2, y_conformalize_2, y_test_2 - ) = train_conformalize_test_split( - X, y, train_size=60, conformalize_size=20, - test_size=20, random_state=RANDOM_STATE - ) - assert np.array_equal(X_train_1, X_train_2) - assert np.array_equal(X_conformalize_1, X_conformalize_2) - assert np.array_equal(X_test_1, X_test_2) - assert np.array_equal(y_train_1, y_train_2) - assert np.array_equal(y_conformalize_1, y_conformalize_2) - assert np.array_equal(y_test_1, y_test_2) - - def test_different_random_state(self, dataset): - X, y = dataset - ( - X_train_1, X_conformalize_1, X_test_1, y_train_1, y_conformalize_1, y_test_1 - ) = train_conformalize_test_split( - X, y, train_size=60, conformalize_size=20, - test_size=20, random_state=RANDOM_STATE - ) - ( - X_train_2, X_conformalize_2, X_test_2, y_train_2, y_conformalize_2, y_test_2 - ) = train_conformalize_test_split( - X, y, train_size=60, conformalize_size=20, - test_size=20, random_state=RANDOM_STATE + 1 - ) - assert not np.array_equal(X_train_1, X_train_2) - assert not np.array_equal(X_conformalize_1, X_conformalize_2) - assert not np.array_equal(X_test_1, X_test_2) - assert not np.array_equal(y_train_1, y_train_2) - assert not np.array_equal(y_conformalize_1, y_conformalize_2) - assert not np.array_equal(y_test_1, y_test_2) - - def test_shuffle_false(self, dataset): - X, y = dataset - ( - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test - ) = train_conformalize_test_split( - X, y, train_size=60, conformalize_size=20, - test_size=20, random_state=RANDOM_STATE, shuffle=False - ) - assert np.array_equal(np.concatenate((y_train, y_conformalize, y_test)), y) - - -@pytest.fixture -def point_predictions(): - return np.array([1, 2, 3]) - - -@pytest.fixture -def point_and_interval_predictions(): - return np.array([1, 2]), np.array([3, 4]) - - -@pytest.mark.parametrize( - "confidence_level, expected", - [ - (0.9, 0.1), - (0.7, 0.3), - (0.999, 0.001), - ] -) -def test_transform_confidence_level_to_alpha(confidence_level, expected): - result = _transform_confidence_level_to_alpha(confidence_level) - assert result == expected - assert str(result) == str(expected) # Ensure clean representation - - -class TestTransformConfidenceLevelToAlphaList: - def test_non_list_iterable(self): - confidence_level = (0.8, 0.7) # Testing a non-list iterable - assert _transform_confidence_level_to_alpha_list(confidence_level) == [0.2, 0.3] - - def test_transform_confidence_level_to_alpha_is_called(self): - with patch( - 'mapie.utils._transform_confidence_level_to_alpha' - ) as mock_transform_confidence_level_to_alpha: - _transform_confidence_level_to_alpha_list([0.2, 0.3]) - mock_transform_confidence_level_to_alpha.assert_called() - - -class TestCheckIfParamInAllowedValues: - def test_error(self): - with pytest.raises(ValueError): - _check_if_param_in_allowed_values("invalid_option", "", ["valid_option"]) - - def test_ok(self): - assert _check_if_param_in_allowed_values("valid", "", ["valid"]) is None - - -def test_check_cv_not_string(): - with pytest.raises(ValueError): - _check_cv_not_string("string") - - -class TestCastPointPredictionsToNdarray: - def test_error(self, point_and_interval_predictions): - with pytest.raises(TypeError): - _cast_point_predictions_to_ndarray(point_and_interval_predictions) - - def test_valid_ndarray(self, point_predictions): - point_predictions = np.array([1, 2, 3]) - result = _cast_point_predictions_to_ndarray(point_predictions) - assert result is point_predictions - assert isinstance(result, np.ndarray) - - -class TestCastPredictionsToNdarrayTuple: - def test_error(self, point_predictions): - with pytest.raises(TypeError): - _cast_predictions_to_ndarray_tuple(point_predictions) - - def test_valid_ndarray(self, point_and_interval_predictions): - result = _cast_predictions_to_ndarray_tuple(point_and_interval_predictions) - assert result is point_and_interval_predictions - assert isinstance(result, tuple) - assert isinstance(result[0], np.ndarray) - assert isinstance(result[1], np.ndarray) - - -@pytest.mark.parametrize( - "params, expected", [(None, {}), ({"a": 1, "b": 2}, {"a": 1, "b": 2})] -) -def test_prepare_params(params, expected): - assert _prepare_params(params) == expected - assert _prepare_params(params) is not params - - -class TestPrepareFitParamsAndSampleWeight: - def test_uses_prepare_params(self): - with patch('mapie.utils._prepare_params') as mock_prepare_params: - _prepare_fit_params_and_sample_weight({"param1": 1}) - mock_prepare_params.assert_called() - - def test_with_sample_weight(self): - fit_params = {"sample_weight": [0.1, 0.2, 0.3]} - assert _prepare_fit_params_and_sample_weight(fit_params) == ( - {}, - [0.1, 0.2, 0.3] - ) - - def test_without_sample_weight(self): - params = {"param1": 1} - assert _prepare_fit_params_and_sample_weight(params) == (params, None) - - -class TestRaiseErrorIfPreviousMethodNotCalled: - def test_raises_error_when_previous_method_not_called(self): - with pytest.raises(ValueError): - _raise_error_if_previous_method_not_called( - "current_method", "previous_method", False - ) - - def test_does_nothing_when_previous_method_called(self): - assert _raise_error_if_previous_method_not_called( - "current_method", "previous_method", True - ) is None - - -class TestRaiseErrorIfMethodAlreadyCalled: - def test_raises_error_when_method_already_called(self): - with pytest.raises(ValueError): - _raise_error_if_method_already_called("method", True) - - def test_does_nothing_when_method_not_called(self): - assert _raise_error_if_method_already_called("method", False) is None - - -class TestRaiseErrorIfFitCalledInPrefitMode: - def test_raises_error_in_prefit_mode(self): - with pytest.raises(ValueError): - _raise_error_if_fit_called_in_prefit_mode(True) - - def test_does_nothing_when_not_in_prefit_mode(self): - assert _raise_error_if_fit_called_in_prefit_mode(False) is None From 9f2ec83e1e40642fdc09e20150f6065d5e28e012 Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Thu, 31 Jul 2025 17:24:53 +0200 Subject: [PATCH 02/14] Merge tests v0 and v1 --- Makefile | 5 +- tests_v1/test_functional/README.md | 10 - .../test_non_regression_classification.py | 419 ------------ .../test_non_regression_regression.py | 645 ------------------ tests_v1/test_unit/README.md | 9 - 5 files changed, 2 insertions(+), 1086 deletions(-) delete mode 100644 tests_v1/test_functional/README.md delete mode 100644 tests_v1/test_functional/test_non_regression_classification.py delete mode 100644 tests_v1/test_functional/test_non_regression_regression.py delete mode 100644 tests_v1/test_unit/README.md diff --git a/Makefile b/Makefile index 14ba2e5af..8414450a7 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ ### Checks that are run in GitHub CI ### lint: - flake8 examples mapie notebooks tests_v1 --max-line-length=88 + flake8 examples mapie notebooks --max-line-length=88 type-check: mypy mapie @@ -14,7 +14,7 @@ coverage: --cov-branch \ --cov=mapie \ --cov-report term-missing \ - --pyargs mapie tests_v1 \ + --pyargs mapie \ --cov-fail-under=100 \ --no-cov-on-fail \ --doctest-modules @@ -37,7 +37,6 @@ all-checks: tests: pytest -vs --doctest-modules mapie - python -m pytest -vs tests_v1 clean-doc: $(MAKE) clean -C doc diff --git a/tests_v1/test_functional/README.md b/tests_v1/test_functional/README.md deleted file mode 100644 index 43da65d73..000000000 --- a/tests_v1/test_functional/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Scope - -Folder for testing the main functionalities of the API as seen from a user point-of-view. - -# Philosophy - -- New tests here should be added wisely. -- Group tests in a class if more than one test is needed for a given functionality. -- Be careful of test time. Testing varied scenarios is more important than trying to test all scenarios. -- Write black-box tests if possible (no mocks): don't test implementation details. diff --git a/tests_v1/test_functional/test_non_regression_classification.py b/tests_v1/test_functional/test_non_regression_classification.py deleted file mode 100644 index 73391ddd8..000000000 --- a/tests_v1/test_functional/test_non_regression_classification.py +++ /dev/null @@ -1,419 +0,0 @@ -import numpy as np -import pytest -from numpy.random import RandomState -from sklearn.datasets import make_classification -from sklearn.ensemble import RandomForestClassifier -from sklearn.linear_model import LogisticRegression -from sklearn.model_selection import LeaveOneOut, GroupKFold - -from mapie.classification import ( - _MapieClassifier, - SplitConformalClassifier, - CrossConformalClassifier, -) -from mapie.conformity_scores import ( - RAPSConformityScore, - APSConformityScore, - TopKConformityScore, - LACConformityScore, -) -from mapie.tests.test_utils import ( - DummyClassifierWithFitAndPredictParams, - train_test_split_shuffle, -) -from numpy.typing import ArrayLike - -RANDOM_STATE = 1 - - -@pytest.fixture(scope="module") -def dataset(): - X, y = make_classification( - n_samples=1000, - n_informative=5, - n_classes=4, - random_state=RANDOM_STATE - ) - sample_weight = RandomState(RANDOM_STATE).random(len(X)) - groups = np.array([i % 5 for i in range(len(X))]) - - ( - X_train, - X_conformalize, - y_train, - y_conformalize, - sample_weight_train, - sample_weight_conformalize, - ) = train_test_split_shuffle( - X, y, random_state=RANDOM_STATE, sample_weight=sample_weight - ) - - return { - "X": X, - "y": y, - "sample_weight": sample_weight, - "groups": groups, - "X_train": X_train, - "X_conformalize": X_conformalize, - "y_train": y_train, - "y_conformalize": y_conformalize, - "sample_weight_train": sample_weight_train, - "sample_weight_conformalize": sample_weight_conformalize, - } - - -@pytest.fixture() -def params_split_test_1(): - return { - "v1": { - "__init__": { - "estimator": LogisticRegression(), - }, - }, - "v0": { - "__init__": { - "estimator": LogisticRegression(), - "conformity_score": LACConformityScore(), - "cv": "prefit" - }, - "predict": { - "alpha": 0.1, - }}} - - -@pytest.fixture() -def params_split_test_2(): - return { - "v1": { - "__init__": { - "estimator": DummyClassifierWithFitAndPredictParams(), - "confidence_level": 0.8, - "prefit": False, - "conformity_score": "top_k", - "random_state": RANDOM_STATE, - }, - "fit": { - "fit_params": {"dummy_fit_param": True}, - }, - "conformalize": { - "predict_params": {"dummy_predict_param": True}, - }}, - "v0": { - "__init__": { - "estimator": DummyClassifierWithFitAndPredictParams(), - "conformity_score": TopKConformityScore(), - "cv": "split", - "random_state": RANDOM_STATE, - }, - "fit": { - "fit_params": {"dummy_fit_param": True}, - "predict_params": {"dummy_predict_param": True}, - }, - "predict": { - "alpha": 0.2, - "dummy_predict_param": True, - }}} - - -@pytest.fixture() -def params_split_test_3(dataset): - return { - "v1": { - "__init__": { - "estimator": RandomForestClassifier(random_state=RANDOM_STATE), - "confidence_level": [0.8, 0.9], - "prefit": False, - "conformity_score": "aps", - "random_state": RANDOM_STATE, - }, - "fit": { - "fit_params": {"sample_weight": dataset["sample_weight_train"]}, - }, - "predict_set": { - "conformity_score_params": {"include_last_label": False} - }}, - "v0": { - "__init__": { - "estimator": RandomForestClassifier(random_state=RANDOM_STATE), - "conformity_score": APSConformityScore(), - "cv": "split", - "random_state": RANDOM_STATE, - }, - "fit": { - "sample_weight": dataset["sample_weight"], - }, - "predict": { - "alpha": [0.2, 0.1], - "include_last_label": False, - }}} - - -@pytest.fixture() -def params_split_test_4(): - return { - "v1": { - "__init__": { - "estimator": LogisticRegression(), - "conformity_score": "raps", - "random_state": RANDOM_STATE, - }}, - "v0": { - "__init__": { - "estimator": LogisticRegression(), - "conformity_score": RAPSConformityScore(), - "cv": "prefit", - "random_state": RANDOM_STATE, - }, - "predict": { - "alpha": 0.1, - }}} - - -@pytest.fixture() -def params_split_test_5(): - return { - "v1": { - "__init__": { - "estimator": LogisticRegression(), - "conformity_score": RAPSConformityScore(size_raps=0.4), - "random_state": RANDOM_STATE, - }}, - "v0": { - "__init__": { - "estimator": LogisticRegression(), - "conformity_score": RAPSConformityScore(size_raps=0.4), - "cv": "prefit", - "random_state": RANDOM_STATE, - }, - "predict": { - "alpha": 0.1, - }}} - - -@pytest.mark.parametrize( - "params_", [ - "params_split_test_1", - "params_split_test_2", - "params_split_test_3", - "params_split_test_4", - "params_split_test_5", - ] -) -def test_split(dataset, params_, request): - X, y, X_train, X_conformalize, y_train, y_conformalize = ( - dataset["X"], - dataset["y"], - dataset["X_train"], - dataset["X_conformalize"], - dataset["y_train"], - dataset["y_conformalize"], - ) - - params = extract_params(request.getfixturevalue(params_)) - - prefit = params["v1_init"].get("prefit", True) - - if prefit: - params["v0_init"]["estimator"].fit(X_train, y_train) - params["v1_init"]["estimator"].fit(X_train, y_train) - - v0 = _MapieClassifier(**params["v0_init"]) - v1 = SplitConformalClassifier(**params["v1_init"]) - - if prefit: - v0.fit(X_conformalize, y_conformalize, **params["v0_fit"]) - else: - v0.fit(X, y, **params["v0_fit"]) - v1.fit(X_train, y_train, **params["v1_fit"]) - v1.conformalize(X_conformalize, y_conformalize, **params["v1_conformalize"]) - - v0_preds, v0_pred_sets = v0.predict(X_conformalize, **params["v0_predict"]) - v1_preds, v1_pred_sets = v1.predict_set(X_conformalize, **params["v1_predict_set"]) - - v1_preds_using_predict: ArrayLike = v1.predict(X_conformalize) - - np.testing.assert_array_equal(v0_preds, v1_preds) - np.testing.assert_array_equal(v0_pred_sets, v1_pred_sets) - np.testing.assert_array_equal(v1_preds_using_predict, v1_preds) - - n_confidence_level = get_number_of_confidence_levels(params["v1_init"]) - - assert v1_pred_sets.shape == ( - len(X_conformalize), - len(np.unique(y)), - n_confidence_level, - ) - - -@pytest.fixture() -def params_cross_test_1(dataset): - return { - "v1": { - "__init__": { - "estimator": LogisticRegression(), - "confidence_level": 0.8, - "conformity_score": "lac", - "cv": 4, - "random_state": RANDOM_STATE, - }, - "fit_conformalize": { - "fit_params": {"sample_weight": dataset["sample_weight"]}, - }, - }, - "v0": { - "__init__": { - "estimator": LogisticRegression(), - "conformity_score": LACConformityScore(), - "cv": 4, - "random_state": RANDOM_STATE, - }, - "fit": { - "sample_weight": dataset["sample_weight"], - }, - "predict": { - "alpha": 0.2, - }}} - - -@pytest.fixture() -def params_cross_test_2(): - return { - "v1": { - "__init__": { - "estimator": DummyClassifierWithFitAndPredictParams(), - "confidence_level": [0.9, 0.8], - "conformity_score": "aps", - "cv": LeaveOneOut(), - "random_state": RANDOM_STATE, - }, - "fit_conformalize": { - "predict_params": {"dummy_predict_param": True}, - }, - "predict_set": { - "conformity_score_params": {"include_last_label": False} - }, - }, - "v0": { - "__init__": { - "estimator": DummyClassifierWithFitAndPredictParams(), - "conformity_score": APSConformityScore(), - "cv": LeaveOneOut(), - "random_state": RANDOM_STATE, - }, - "fit": { - "predict_params": {"dummy_predict_param": True}, - }, - "predict": { - "alpha": [0.1, 0.2], - "include_last_label": False, - "dummy_predict_param": True, - }}} - - -@pytest.fixture() -def params_cross_test_3(dataset): - return { - "v1": { - "__init__": { - "estimator": DummyClassifierWithFitAndPredictParams(), - "cv": GroupKFold(), - "random_state": RANDOM_STATE, - }, - "fit_conformalize": { - "groups": dataset["groups"], - "fit_params": {"dummy_fit_param": True}, - }, - "predict_set": { - "agg_scores": "crossval", - }, - }, - "v0": { - "__init__": { - "estimator": DummyClassifierWithFitAndPredictParams(), - "cv": GroupKFold(), - "random_state": RANDOM_STATE, - }, - "fit": { - "groups": dataset["groups"], - "fit_params": {"dummy_fit_param": True}, - }, - "predict": { - "alpha": 0.1, - "agg_scores": "crossval", - }}} - - -@pytest.fixture() -def params_cross_test_4(): - return { - "v1": { - "__init__": { - "estimator": RandomForestClassifier(random_state=RANDOM_STATE), - "confidence_level": 0.7, - "conformity_score": LACConformityScore(), - "random_state": RANDOM_STATE, - }, - }, - "v0": { - "__init__": { - "estimator": RandomForestClassifier(random_state=RANDOM_STATE), - "cv": 5, - "random_state": RANDOM_STATE, - }, - "predict": { - "alpha": 0.3, - }}} - - -@pytest.mark.parametrize( - "params_", [ - "params_cross_test_1", - "params_cross_test_2", - "params_cross_test_3", - "params_cross_test_4", - ] -) -def test_cross(dataset, params_, request): - X, y = dataset["X"], dataset["y"] - - params = extract_params(request.getfixturevalue(params_)) - - v0 = _MapieClassifier(**params["v0_init"]) - v1 = CrossConformalClassifier(**params["v1_init"]) - - v0.fit(X, y, **params["v0_fit"]) - v1.fit_conformalize(X, y, **params["v1_fit_conformalize"]) - - v0_preds, v0_pred_sets = v0.predict(X, **params["v0_predict"]) - v1_preds, v1_pred_sets = v1.predict_set(X, **params["v1_predict_set"]) - - v1_preds_using_predict: ArrayLike = v1.predict(X) - - np.testing.assert_array_equal(v0_preds, v1_preds) - np.testing.assert_array_equal(v0_pred_sets, v1_pred_sets) - np.testing.assert_array_equal(v1_preds_using_predict, v1_preds) - - n_confidence_level = get_number_of_confidence_levels(params["v1_init"]) - assert v1_pred_sets.shape == ( - len(X), - len(np.unique(y)), - n_confidence_level, - ) - - -def extract_params(params): - return { - "v0_init": params["v0"].get("__init__", {}), - "v0_fit": params["v0"].get("fit", {}), - "v0_predict": params["v0"].get("predict", {}), - "v1_init": params["v1"].get("__init__", {}), - "v1_fit": params["v1"].get("fit", {}), - "v1_conformalize": params["v1"].get("conformalize", {}), - "v1_predict_set": params["v1"].get("predict_set", {}), - "v1_fit_conformalize": params["v1"].get("fit_conformalize", {}) - } - - -def get_number_of_confidence_levels(v1_init_params): - confidence_level = v1_init_params.get("confidence_level", 0.9) - return 1 if isinstance(confidence_level, float) else len(confidence_level) diff --git a/tests_v1/test_functional/test_non_regression_regression.py b/tests_v1/test_functional/test_non_regression_regression.py deleted file mode 100644 index a9dec18f6..000000000 --- a/tests_v1/test_functional/test_non_regression_regression.py +++ /dev/null @@ -1,645 +0,0 @@ -from __future__ import annotations -from typing import Optional, Union, Dict, Type - -import numpy as np -import pytest -from numpy.random import RandomState -from sklearn.compose import TransformedTargetRegressor -from sklearn.datasets import make_regression -from sklearn.linear_model import LinearRegression -from sklearn.linear_model import QuantileRegressor -from sklearn.ensemble import GradientBoostingRegressor -from sklearn.model_selection import train_test_split - -from mapie.subsample import Subsample -from numpy.typing import ArrayLike, NDArray -from mapie.conformity_scores import GammaConformityScore, \ - AbsoluteConformityScore, ResidualNormalisedScore -from mapie.regression import SplitConformalRegressor, \ - CrossConformalRegressor, \ - JackknifeAfterBootstrapRegressor, \ - ConformalizedQuantileRegressor - -from mapie.regression.regression import _MapieRegressor -from mapie.regression.quantile_regression import _MapieQuantileRegressor -from mapie.tests.test_utils import filter_params, train_test_split_shuffle -from sklearn.model_selection import LeaveOneOut, GroupKFold - -RANDOM_STATE = 1 -K_FOLDS = 3 -N_BOOTSTRAPS = 30 -N_SAMPLES = 200 -N_GROUPS = 5 - -X, y_signed = make_regression( - n_samples=N_SAMPLES, - n_features=10, - noise=1.0, - random_state=RANDOM_STATE -) -y = np.abs(y_signed) -sample_weight = RandomState(RANDOM_STATE).random(len(X)) -groups = [j for j in range(N_GROUPS) for i in range((N_SAMPLES//N_GROUPS))] -positive_predictor = TransformedTargetRegressor( - regressor=LinearRegression(), - func=lambda y_: np.log(y_ + 1), - inverse_func=lambda X_: np.exp(X_) - 1 -) - -sample_weight_train = train_test_split( - X, - y, - sample_weight, - test_size=0.4, - random_state=RANDOM_STATE -)[-2] - - -params_test_cases_cross = [ - { - "v1": { - "class": CrossConformalRegressor, - "__init__": { - "confidence_level": 0.8, - "conformity_score": "absolute", - "cv": 4, - "method": "base", - "random_state": RANDOM_STATE, - }, - "fit_conformalize": { - "fit_params": {"sample_weight": sample_weight}, - }, - "predict_interval": { - "aggregate_predictions": "median", - }, - "predict": { - "aggregate_predictions": "median", - } - }, - "v0": { - "__init__": { - "conformity_score": AbsoluteConformityScore(), - "cv": 4, - "method": "base", - "random_state": RANDOM_STATE, - "agg_function": "median", - }, - "fit": { - "sample_weight": sample_weight, - }, - "predict": { - "alpha": 0.2, - "ensemble": True, - }, - }, - }, - { - "v1": { - "class": CrossConformalRegressor, - "__init__": { - "estimator": positive_predictor, - "confidence_level": [0.5, 0.5], - "conformity_score": "gamma", - "cv": LeaveOneOut(), - "method": "plus", - "random_state": RANDOM_STATE, - }, - "predict_interval": { - "minimize_interval_width": True, - }, - }, - "v0": { - "__init__": { - "estimator": positive_predictor, - "conformity_score": GammaConformityScore(), - "cv": LeaveOneOut(), - "agg_function": "mean", - "method": "plus", - "random_state": RANDOM_STATE, - }, - "predict": { - "alpha": [0.5, 0.5], - "optimize_beta": True, - "ensemble": True, - }, - }, - }, - { - "v1": { - "class": CrossConformalRegressor, - "__init__": { - "cv": GroupKFold(), - "method": "minmax", - "random_state": RANDOM_STATE, - }, - "fit_conformalize": { - "groups": groups, - }, - "predict_interval": { - "allow_infinite_bounds": True, - "aggregate_predictions": None, - }, - "predict": { - "aggregate_predictions": None, - }, - }, - "v0": { - "__init__": { - "cv": GroupKFold(), - "method": "minmax", - "random_state": RANDOM_STATE, - }, - "fit": { - "groups": groups, - }, - "predict": { - "alpha": 0.1, - "allow_infinite_bounds": True, - }, - }, - }, -] - -params_test_cases_jackknife = [ - { - "v1": { - "class": JackknifeAfterBootstrapRegressor, - "__init__": { - "confidence_level": 0.8, - "conformity_score": "absolute", - "resampling": Subsample( - n_resamplings=15, random_state=RANDOM_STATE - ), - "aggregation_method": "median", - "method": "plus", - "random_state": RANDOM_STATE, - }, - "fit_conformalize": { - "fit_params": {"sample_weight": sample_weight}, - }, - }, - "v0": { - "__init__": { - "conformity_score": AbsoluteConformityScore(), - "cv": Subsample(n_resamplings=15, random_state=RANDOM_STATE), - "agg_function": "median", - "method": "plus", - "random_state": RANDOM_STATE, - }, - "fit": { - "sample_weight": sample_weight, - }, - "predict": { - "alpha": 0.2, - "ensemble": True, - }, - }, - }, - { - "v1": { - "class": JackknifeAfterBootstrapRegressor, - "__init__": { - "estimator": positive_predictor, - "confidence_level": [0.5, 0.5], - "aggregation_method": "mean", - "conformity_score": "gamma", - "resampling": Subsample( - n_resamplings=20, - replace=True, - random_state=RANDOM_STATE - ), - "method": "plus", - "random_state": RANDOM_STATE, - }, - "predict_interval": { - "minimize_interval_width": True, - }, - }, - "v0": { - "__init__": { - "estimator": positive_predictor, - "conformity_score": GammaConformityScore(), - "agg_function": "mean", - "cv": Subsample( - n_resamplings=20, - replace=True, - random_state=RANDOM_STATE - ), - "method": "plus", - "random_state": RANDOM_STATE, - }, - "predict": { - "alpha": [0.5, 0.5], - "optimize_beta": True, - "ensemble": True, - }, - }, - }, - { - "v1": { - "class": JackknifeAfterBootstrapRegressor, - "__init__": { - "confidence_level": 0.9, - "resampling": Subsample( - n_resamplings=30, random_state=RANDOM_STATE - ), - "method": "minmax", - "aggregation_method": "mean", - "random_state": RANDOM_STATE, - }, - "predict_interval": { - "allow_infinite_bounds": True, - }, - }, - "v0": { - "__init__": { - "cv": Subsample(n_resamplings=30, random_state=RANDOM_STATE), - "method": "minmax", - "agg_function": "mean", - "random_state": RANDOM_STATE, - }, - "predict": { - "alpha": 0.1, - "ensemble": True, - "allow_infinite_bounds": True, - }, - }, - }, -] - - -def run_v0_pipeline_cross_or_jackknife(params): - params_ = params["v0"] - mapie_regressor = _MapieRegressor(**params_.get("__init__", {})) - - mapie_regressor.fit(X, y, **params_.get("fit", {})) - preds, pred_intervals = mapie_regressor.predict(X, **params_.get("predict", {})) - - return preds, pred_intervals - - -def run_v1_pipeline_cross_or_jackknife(params): - params_ = params["v1"] - init_params = params_.get("__init__", {}) - confidence_level = init_params.get("confidence_level", 0.9) - confidence_level_length = 1 if isinstance(confidence_level, float) else len( - confidence_level - ) - minimize_interval_width = params_.get("predict_interval", {}).get( - "minimize_interval_width" - ) - - mapie_regressor = params_["class"](**init_params) - mapie_regressor.fit_conformalize(X, y, **params_.get("fit_conformalize", {})) - - X_test = X - preds, pred_intervals = mapie_regressor.predict_interval( - X_test, - **params_.get("predict_interval", {}) - ) - preds_using_predict = mapie_regressor.predict( - X_test, - **params_.get("predict", {}) - ) - - return ( - preds, - pred_intervals, - preds_using_predict, - len(X_test), - confidence_level_length, - minimize_interval_width, - ) - - -@pytest.mark.parametrize( - "params", - params_test_cases_cross + params_test_cases_jackknife -) -def test_cross_and_jackknife(params: dict) -> None: - v0_preds, v0_pred_intervals = run_v0_pipeline_cross_or_jackknife(params) - ( - v1_preds, - v1_pred_intervals, - v1_preds_using_predict, - X_test_length, - confidence_level_length, - minimize_interval_width, - ) = run_v1_pipeline_cross_or_jackknife(params) - - np.testing.assert_array_equal(v0_preds, v1_preds) - np.testing.assert_array_equal(v0_pred_intervals, v1_pred_intervals) - np.testing.assert_array_equal(v1_preds_using_predict, v1_preds) - - if not minimize_interval_width: - # condition to remove when optimize_beta/minimize_interval_width works - # but keep assertion to check shapes - assert v1_pred_intervals.shape == (X_test_length, 2, confidence_level_length) - - -# Below are SplitConformalRegressor and ConformalizedQuantileRegressor tests -# They're using an outdated structure, prefer the style used for CrossConformalRegressor -# and JackknifeAfterBootstrapRegressor above - -params_test_cases_split = [ - { - "v0": { - "alpha": 0.2, - "conformity_score": AbsoluteConformityScore(), - "cv": "split", - "test_size": 0.4, - "sample_weight": sample_weight, - "random_state": RANDOM_STATE, - }, - "v1": { - "confidence_level": 0.8, - "conformity_score": "absolute", - "prefit": False, - "test_size": 0.4, - "fit_params": {"sample_weight": sample_weight_train}, - } - }, - { - "v0": { - "estimator": positive_predictor, - "test_size": 0.2, - "alpha": [0.5, 0.5], - "conformity_score": GammaConformityScore(), - "cv": "split", - "random_state": RANDOM_STATE, - }, - "v1": { - "estimator": positive_predictor, - "test_size": 0.2, - "confidence_level": [0.5, 0.5], - "conformity_score": "gamma", - "prefit": False, - } - }, - { - "v0": { - "estimator": LinearRegression(), - "alpha": 0.1, - "test_size": 0.2, - "conformity_score": ResidualNormalisedScore( - random_state=RANDOM_STATE - ), - "cv": "prefit", - "allow_infinite_bounds": True, - "random_state": RANDOM_STATE, - }, - "v1": { - "estimator": LinearRegression(), - "confidence_level": 0.9, - "prefit": True, - "test_size": 0.2, - "conformity_score": ResidualNormalisedScore( - random_state=RANDOM_STATE - ), - "allow_infinite_bounds": True, - } - }, - { - "v0": { - "estimator": positive_predictor, - "alpha": 0.1, - "conformity_score": GammaConformityScore(), - "cv": "split", - "random_state": RANDOM_STATE, - "test_size": 0.3, - "optimize_beta": True - }, - "v1": { - "estimator": positive_predictor, - "confidence_level": 0.9, - "prefit": False, - "conformity_score": GammaConformityScore(), - "test_size": 0.3, - "minimize_interval_width": True - } - }, -] - - -@pytest.mark.parametrize("params_split", params_test_cases_split) -def test_intervals_and_predictions_exact_equality_split(params_split: dict) -> None: - v0_params = params_split["v0"] - v1_params = params_split["v1"] - - test_size = v1_params.get("test_size", None) - prefit = v1_params.get("prefit", False) - - compare_model_predictions_and_intervals( - model_v0=_MapieRegressor, - model_v1=SplitConformalRegressor, - X=X, - y=y, - v0_params=v0_params, - v1_params=v1_params, - test_size=test_size, - prefit=prefit, - random_state=RANDOM_STATE, - ) - - -split_model = QuantileRegressor( - solver="highs-ds", - alpha=0.0, - ) - -gbr_models = [] -gbr_alpha = 0.1 - -for alpha_ in [gbr_alpha / 2, (1 - (gbr_alpha / 2)), 0.5]: - estimator_ = GradientBoostingRegressor( - loss='quantile', - alpha=alpha_, - n_estimators=100, - learning_rate=0.1, - max_depth=3 - ) - gbr_models.append(estimator_) - -params_test_cases_quantile = [ - { - "v0": { - "alpha": 0.2, - "cv": "split", - "method": "quantile", - "calib_size": 0.4, - "sample_weight": sample_weight, - "random_state": RANDOM_STATE, - "symmetry": False, - }, - "v1": { - "confidence_level": 0.8, - "prefit": False, - "test_size": 0.4, - "fit_params": {"sample_weight": sample_weight_train}, - }, - }, - { - "v0": { - "estimator": gbr_models, - "alpha": gbr_alpha, - "cv": "prefit", - "method": "quantile", - "calib_size": 0.2, - "sample_weight": sample_weight, - "optimize_beta": True, - "random_state": RANDOM_STATE, - }, - "v1": { - "estimator": gbr_models, - "confidence_level": 1-gbr_alpha, - "prefit": True, - "test_size": 0.2, - "fit_params": {"sample_weight": sample_weight}, - "minimize_interval_width": True, - "symmetric_correction": True, - }, - }, - { - "v0": { - "estimator": split_model, - "alpha": 0.5, - "cv": "split", - "method": "quantile", - "calib_size": 0.3, - "allow_infinite_bounds": True, - "random_state": RANDOM_STATE, - "symmetry": False, - }, - "v1": { - "estimator": split_model, - "confidence_level": 0.5, - "prefit": False, - "test_size": 0.3, - "allow_infinite_bounds": True, - }, - }, - { - "v0": { - "alpha": 0.1, - "cv": "split", - "method": "quantile", - "calib_size": 0.3, - "random_state": RANDOM_STATE, - }, - "v1": { - "confidence_level": 0.9, - "prefit": False, - "test_size": 0.3, - "symmetric_correction": True, - }, - }, -] - - -@pytest.mark.parametrize("params_quantile", params_test_cases_quantile) -def test_intervals_and_predictions_exact_equality_quantile( - params_quantile: dict -) -> None: - v0_params = params_quantile["v0"] - v1_params = params_quantile["v1"] - - test_size = v1_params.get("test_size", None) - prefit = v1_params.get("prefit", False) - - compare_model_predictions_and_intervals( - model_v0=_MapieQuantileRegressor, - model_v1=ConformalizedQuantileRegressor, - X=X, - y=y, - v0_params=v0_params, - v1_params=v1_params, - test_size=test_size, - prefit=prefit, - random_state=RANDOM_STATE, - ) - - -def compare_model_predictions_and_intervals( - model_v0: Type[_MapieRegressor], - model_v1: Type[Union[ - SplitConformalRegressor, - CrossConformalRegressor, - JackknifeAfterBootstrapRegressor, - ConformalizedQuantileRegressor - ]], - X: NDArray, - y: NDArray, - v0_params: Dict = {}, - v1_params: Dict = {}, - prefit: bool = False, - test_size: Optional[float] = None, - random_state: int = RANDOM_STATE, -) -> None: - if v0_params.get("alpha"): - if isinstance(v0_params["alpha"], float): - n_alpha = 1 - else: - n_alpha = len(v0_params["alpha"]) - else: - n_alpha = 1 - - if test_size is not None: - X_train, X_conf, y_train, y_conf = train_test_split_shuffle( - X, - y, - test_size=test_size, - random_state=random_state, - ) - else: - X_train, X_conf, y_train, y_conf = X, X, y, y - - if prefit: - estimator = v0_params["estimator"] - if isinstance(estimator, list): - for single_estimator in estimator: - single_estimator.fit(X_train, y_train) - else: - estimator.fit(X_train, y_train) - - v0_params["estimator"] = estimator - v1_params["estimator"] = estimator - - v0_init_params = filter_params(model_v0.__init__, v0_params) - v1_init_params = filter_params(model_v1.__init__, v1_params) - - v0 = model_v0(**v0_init_params) - v1 = model_v1(**v1_init_params) - - v0_fit_params = filter_params(v0.fit, v0_params) - v1_fit_params = filter_params(v1.fit, v1_params) - v1_conformalize_params = filter_params(v1.conformalize, v1_params) - - if prefit: - v0.fit(X_conf, y_conf, **v0_fit_params) - else: - v0.fit(X, y, **v0_fit_params) - v1.fit(X_train, y_train, **v1_fit_params) - - v1.conformalize(X_conf, y_conf, **v1_conformalize_params) - - v0_predict_params = filter_params(v0.predict, v0_params) - if 'alpha' in v0_init_params: - v0_predict_params.pop('alpha') - - v1_predict_params = filter_params(v1.predict, v1_params) - v1_predict_interval_params = filter_params(v1.predict_interval, v1_params) - - v0_preds, v0_pred_intervals = v0.predict(X_conf, **v0_predict_params) - v1_preds, v1_pred_intervals = v1.predict_interval( - X_conf, **v1_predict_interval_params - ) - - v1_preds_using_predict: ArrayLike = v1.predict(X_conf, **v1_predict_params) - - np.testing.assert_array_equal(v0_preds, v1_preds) - np.testing.assert_array_equal(v0_pred_intervals, v1_pred_intervals) - np.testing.assert_array_equal(v1_preds_using_predict, v1_preds) - if not v0_params.get("optimize_beta"): - # condition to remove when optimize_beta works - # keep assertion - assert v1_pred_intervals.shape == (len(X_conf), 2, n_alpha) diff --git a/tests_v1/test_unit/README.md b/tests_v1/test_unit/README.md deleted file mode 100644 index 51c84eeab..000000000 --- a/tests_v1/test_unit/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Scope - -Folder for testing small functions ("unit" = function). - -# Philosophy - -- Group tests in a class if more than one test is needed for a given function. -- Focus on the function goal to define the test cases (the function name is a good hint). -- Prefer black-box tests (no mocks) if possible, to avoid testing implementation details. From 00c73abe91ddbb147c08e1abf5bae63e3d1dce9d Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Fri, 1 Aug 2025 11:07:38 +0200 Subject: [PATCH 03/14] Merge tests v0 and v1 refacto --- mapie/tests/test_classification.py | 284 +++++----- mapie/tests/test_common.py | 77 ++- mapie/tests/test_conformity_scores_utils.py | 44 +- mapie/tests/test_regression.py | 394 +++++++------- mapie/tests/test_utils.py | 561 +++++++++----------- 5 files changed, 680 insertions(+), 680 deletions(-) diff --git a/mapie/tests/test_classification.py b/mapie/tests/test_classification.py index a5e920ec7..9f96460ca 100644 --- a/mapie/tests/test_classification.py +++ b/mapie/tests/test_classification.py @@ -37,6 +37,148 @@ from mapie.utils import check_proba_normalized from mapie.metrics.classification import classification_coverage_score +@pytest.fixture(scope="module") +def dataset_classification(): + X, y = make_classification( + n_samples=500, n_informative=5, n_classes=4, random_state=random_state, + ) + X_train, X_conf_test, y_train, y_conf_test = train_test_split( + X, y, random_state=random_state + ) + X_conformalize, X_test, y_conformalize, y_test = train_test_split( + X_conf_test, y_conf_test, random_state=random_state + ) + return X_train, X_conformalize, X_test, y_train, y_conformalize, y_test + + +@pytest.mark.parametrize( + "split_technique,predict_method,dataset,estimator_class", + [ + ( + SplitConformalClassifier, + "predict_set", + "dataset_classification", + DummyClassifier + ) + ] +) +class TestWrongMethodsOrderRaisesErrorForSplitTechniquesClassification: + def test_with_prefit_false( + self, + split_technique, + predict_method, + dataset, + estimator_class, + request + ): + dataset = request.getfixturevalue(dataset) + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset + estimator = estimator_class() + technique = split_technique(estimator=estimator, prefit=False) + + with pytest.raises(ValueError, match=r"call fit before calling conformalize"): + technique.conformalize( + X_conformalize, + y_conformalize + ) + + technique.fit(X_train, y_train) + + with pytest.raises(ValueError, match=r"fit method already called"): + technique.fit(X_train, y_train) + with pytest.raises( + ValueError, + match=r"call conformalize before calling predict" + ): + technique.predict(X_test) + + with pytest.raises( + ValueError, + match=f"call conformalize before calling {predict_method}" + ): + getattr(technique, predict_method)(X_test) + + technique.conformalize(X_conformalize, y_conformalize) + + with pytest.raises(ValueError, match=r"conformalize method already called"): + technique.conformalize(X_conformalize, y_conformalize) + + def test_with_prefit_true( + self, + split_technique, + predict_method, + dataset, + estimator_class, + request + ): + dataset = request.getfixturevalue(dataset) + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset + estimator = estimator_class() + estimator.fit(X_train, y_train) + + technique = split_technique(estimator=estimator, prefit=True) + + with pytest.raises(ValueError, match=r"The fit method must be skipped"): + technique.fit(X_train, y_train) + with pytest.raises( + ValueError, + match=r"call conformalize before calling predict" + ): + technique.predict(X_test) + + with pytest.raises( + ValueError, + match=f"call conformalize before calling {predict_method}" + ): + getattr(technique, predict_method)(X_test) + + technique.conformalize(X_conformalize, y_conformalize) + + with pytest.raises(ValueError, match=r"conformalize method already called"): + technique.conformalize(X_conformalize, y_conformalize) + + +@pytest.mark.parametrize( + "cross_technique,predict_method,dataset,estimator_class", + [ + ( + CrossConformalClassifier, + "predict_set", + "dataset_classification", + DummyClassifier + ) + ] +) +class TestWrongMethodsOrderRaisesErrorForCrossTechniques: + def test_wrong_methods_order( + self, + cross_technique, + predict_method, + dataset, + estimator_class, + request + ): + dataset = request.getfixturevalue(dataset) + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset + technique = cross_technique(estimator=estimator_class()) + + with pytest.raises( + ValueError, + match=r"call fit_conformalize before calling predict" + ): + technique.predict(X_test) + with pytest.raises( + ValueError, + match=f"call fit_conformalize before calling {predict_method}" + ): + getattr(technique, predict_method)(X_test) + + technique.fit_conformalize(X_conformalize, y_conformalize) + + with pytest.raises(ValueError, match=r"fit_conformalize method already called"): + technique.fit_conformalize(X_conformalize, y_conformalize) + + random_state = 42 WRONG_INCLUDE_LABELS = ["randomised", "True", "False", "other", 1, 2.5, (1, 2)] @@ -2030,145 +2172,3 @@ def test_using_one_predict_parameter_into_fit_but_not_in_predict() -> None: r"is used in the predict method as called in the fit." )): mapie_fitted.predict(X_test) - - -@pytest.fixture(scope="module") -def dataset_classification(): - X, y = make_classification( - n_samples=500, n_informative=5, n_classes=4, random_state=random_state, - ) - X_train, X_conf_test, y_train, y_conf_test = train_test_split( - X, y, random_state=random_state - ) - X_conformalize, X_test, y_conformalize, y_test = train_test_split( - X_conf_test, y_conf_test, random_state=random_state - ) - return X_train, X_conformalize, X_test, y_train, y_conformalize, y_test - - -@pytest.mark.parametrize( - "split_technique,predict_method,dataset,estimator_class", - [ - ( - SplitConformalClassifier, - "predict_set", - "dataset_classification", - DummyClassifier - ) - ] -) -class TestWrongMethodsOrderRaisesErrorForSplitTechniquesClassification: - def test_with_prefit_false( - self, - split_technique, - predict_method, - dataset, - estimator_class, - request - ): - dataset = request.getfixturevalue(dataset) - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset - estimator = estimator_class() - technique = split_technique(estimator=estimator, prefit=False) - - with pytest.raises(ValueError, match=r"call fit before calling conformalize"): - technique.conformalize( - X_conformalize, - y_conformalize - ) - - technique.fit(X_train, y_train) - - with pytest.raises(ValueError, match=r"fit method already called"): - technique.fit(X_train, y_train) - with pytest.raises( - ValueError, - match=r"call conformalize before calling predict" - ): - technique.predict(X_test) - - with pytest.raises( - ValueError, - match=f"call conformalize before calling {predict_method}" - ): - getattr(technique, predict_method)(X_test) - - technique.conformalize(X_conformalize, y_conformalize) - - with pytest.raises(ValueError, match=r"conformalize method already called"): - technique.conformalize(X_conformalize, y_conformalize) - - def test_with_prefit_true( - self, - split_technique, - predict_method, - dataset, - estimator_class, - request - ): - dataset = request.getfixturevalue(dataset) - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset - estimator = estimator_class() - estimator.fit(X_train, y_train) - - technique = split_technique(estimator=estimator, prefit=True) - - with pytest.raises(ValueError, match=r"The fit method must be skipped"): - technique.fit(X_train, y_train) - with pytest.raises( - ValueError, - match=r"call conformalize before calling predict" - ): - technique.predict(X_test) - - with pytest.raises( - ValueError, - match=f"call conformalize before calling {predict_method}" - ): - getattr(technique, predict_method)(X_test) - - technique.conformalize(X_conformalize, y_conformalize) - - with pytest.raises(ValueError, match=r"conformalize method already called"): - technique.conformalize(X_conformalize, y_conformalize) - - -@pytest.mark.parametrize( - "cross_technique,predict_method,dataset,estimator_class", - [ - ( - CrossConformalClassifier, - "predict_set", - "dataset_classification", - DummyClassifier - ) - ] -) -class TestWrongMethodsOrderRaisesErrorForCrossTechniques: - def test_wrong_methods_order( - self, - cross_technique, - predict_method, - dataset, - estimator_class, - request - ): - dataset = request.getfixturevalue(dataset) - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset - technique = cross_technique(estimator=estimator_class()) - - with pytest.raises( - ValueError, - match=r"call fit_conformalize before calling predict" - ): - technique.predict(X_test) - with pytest.raises( - ValueError, - match=f"call fit_conformalize before calling {predict_method}" - ): - getattr(technique, predict_method)(X_test) - - technique.fit_conformalize(X_conformalize, y_conformalize) - - with pytest.raises(ValueError, match=r"fit_conformalize method already called"): - technique.fit_conformalize(X_conformalize, y_conformalize) diff --git a/mapie/tests/test_common.py b/mapie/tests/test_common.py index 39156694e..126926699 100644 --- a/mapie/tests/test_common.py +++ b/mapie/tests/test_common.py @@ -1,19 +1,90 @@ +import inspect from inspect import signature -from typing import Any, List, Tuple +from typing import Any, List, Tuple, Optional, Union, Callable, Dict import numpy as np import pytest -from sklearn.base import BaseEstimator +from numpy._typing import NDArray, ArrayLike +from sklearn.base import BaseEstimator, ClassifierMixin from sklearn.exceptions import NotFittedError from sklearn.linear_model import LinearRegression, LogisticRegression -from sklearn.model_selection import KFold +from sklearn.model_selection import KFold, ShuffleSplit from sklearn.pipeline import make_pipeline from sklearn.utils.validation import check_is_fitted +from typing_extensions import Self from mapie.classification import _MapieClassifier from mapie.regression.regression import _MapieRegressor from mapie.regression.quantile_regression import _MapieQuantileRegressor +def train_test_split_shuffle( + X: NDArray, + y: NDArray, + test_size: Optional[float] = None, + random_state: int = 42, + sample_weight: Optional[NDArray] = None, +) -> Union[Tuple[Any, Any, Any, Any], Tuple[Any, Any, Any, Any, Any, Any]]: + splitter = ShuffleSplit( + n_splits=1, + test_size=test_size, + random_state=random_state + ) + train_idx, test_idx = next(splitter.split(X)) + + X_train, X_test = X[train_idx], X[test_idx] + y_train, y_test = y[train_idx], y[test_idx] + if sample_weight is not None: + sample_weight_train = sample_weight[train_idx] + sample_weight_test = sample_weight[test_idx] + return X_train, X_test, y_train, y_test, sample_weight_train, sample_weight_test + + return X_train, X_test, y_train, y_test + + +def filter_params( + function: Callable, + params: Optional[Dict[str, Any]] = None +) -> Dict[str, Any]: + if params is None: + return {} + + model_params = inspect.signature(function).parameters + return {k: v for k, v in params.items() if k in model_params} + + +class DummyClassifierWithFitAndPredictParams(BaseEstimator, ClassifierMixin): + def __init__(self): + self.classes_ = None + self._dummy_fit_param = None + + def fit(self, X: ArrayLike, y: ArrayLike, dummy_fit_param: bool = False) -> Self: + self.classes_ = np.unique(y) + if len(self.classes_) < 2: + raise ValueError("Dummy classifier needs at least 3 classes") + self._dummy_fit_param = dummy_fit_param + return self + + def predict_proba(self, X: ArrayLike, dummy_predict_param: bool = False) -> NDArray: + probas = np.zeros((len(X), len(self.classes_))) + if self._dummy_fit_param & dummy_predict_param: + probas[:, 0] = 0.1 + probas[:, 1] = 0.9 + elif self._dummy_fit_param: + probas[:, 1] = 0.1 + probas[:, 2] = 0.9 + elif dummy_predict_param: + probas[:, 1] = 0.1 + probas[:, 0] = 0.9 + else: + probas[:, 2] = 0.1 + probas[:, 0] = 0.9 + return probas + + def predict(self, X: ArrayLike, dummy_predict_param: bool = False) -> NDArray: + y_preds_proba = self.predict_proba(X, dummy_predict_param) + return np.amax(y_preds_proba, axis=0) + + X_toy = np.arange(18).reshape(-1, 1) y_toy = np.array( [0, 0, 1, 0, 1, 2, 1, 2, 2, 0, 0, 1, 0, 1, 2, 1, 2, 2] diff --git a/mapie/tests/test_conformity_scores_utils.py b/mapie/tests/test_conformity_scores_utils.py index df71c06ab..2e7bf342d 100644 --- a/mapie/tests/test_conformity_scores_utils.py +++ b/mapie/tests/test_conformity_scores_utils.py @@ -11,6 +11,28 @@ from mapie.conformity_scores.utils import check_and_select_conformity_score +class TestCheckAndSelectConformityScore: + + @pytest.mark.parametrize( + "score, score_type, expected_class", [ + (AbsoluteConformityScore(), BaseRegressionScore, AbsoluteConformityScore), + ("gamma", BaseRegressionScore, GammaConformityScore), + (LACConformityScore(), BaseClassificationScore, LACConformityScore), + ("top_k", BaseClassificationScore, TopKConformityScore), + ] + ) + def test_with_valid_inputs(self, score, score_type, expected_class): + result = check_and_select_conformity_score(score, score_type) + assert isinstance(result, expected_class) + + @pytest.mark.parametrize( + "score_type", [BaseRegressionScore, BaseClassificationScore] + ) + def test_with_invalid_input(self, score_type): + with pytest.raises(ValueError): + check_and_select_conformity_score("I'm not a valid input :(", score_type) + + Y_TRUE_PROBA_PLACE = [ [ np.array([2, 0]), @@ -55,25 +77,3 @@ def test_get_true_label_position( found_place = get_true_label_position(y_pred_proba, y_true) assert (found_place == place).all() - - -class TestCheckAndSelectConformityScore: - - @pytest.mark.parametrize( - "score, score_type, expected_class", [ - (AbsoluteConformityScore(), BaseRegressionScore, AbsoluteConformityScore), - ("gamma", BaseRegressionScore, GammaConformityScore), - (LACConformityScore(), BaseClassificationScore, LACConformityScore), - ("top_k", BaseClassificationScore, TopKConformityScore), - ] - ) - def test_with_valid_inputs(self, score, score_type, expected_class): - result = check_and_select_conformity_score(score, score_type) - assert isinstance(result, expected_class) - - @pytest.mark.parametrize( - "score_type", [BaseRegressionScore, BaseClassificationScore] - ) - def test_with_invalid_input(self, score_type): - with pytest.raises(ValueError): - check_and_select_conformity_score("I'm not a valid input :(", score_type) diff --git a/mapie/tests/test_regression.py b/mapie/tests/test_regression.py index 379fdea35..f99cd4358 100644 --- a/mapie/tests/test_regression.py +++ b/mapie/tests/test_regression.py @@ -38,6 +38,203 @@ JackknifeAfterBootstrapRegressor, SplitConformalRegressor, CrossConformalRegressor from mapie.subsample import Subsample +class TestCheckAndConvertResamplingToCv: + def test_with_integer(self): + regressor = JackknifeAfterBootstrapRegressor() + cv = regressor._check_and_convert_resampling_to_cv(50) + + assert isinstance(cv, Subsample) + assert cv.n_resamplings == 50 + + def test_with_subsample(self): + custom_subsample = Subsample(n_resamplings=25, random_state=42) + regressor = JackknifeAfterBootstrapRegressor() + cv = regressor._check_and_convert_resampling_to_cv(custom_subsample) + + assert cv is custom_subsample + + def test_with_invalid_input(self): + regressor = JackknifeAfterBootstrapRegressor() + + with pytest.raises( + ValueError, + match="resampling must be an integer or a Subsample instance" + ): + regressor._check_and_convert_resampling_to_cv("invalid_input") + + +@pytest.fixture(scope="module") +def dataset_regression(): + X, y = make_regression( + n_samples=500, n_features=2, noise=1.0, random_state=random_state + ) + X_train, X_conf_test, y_train, y_conf_test = train_test_split( + X, y, random_state=random_state + ) + X_conformalize, X_test, y_conformalize, y_test = train_test_split( + X_conf_test, y_conf_test, random_state=random_state + ) + return X_train, X_conformalize, X_test, y_train, y_conformalize, y_test + + +def test_scr_same_predictions_prefit_not_prefit(dataset_regression) -> None: + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = ( + dataset_regression) + regressor = LinearRegression() + regressor.fit(X_train, y_train) + scr_prefit = SplitConformalRegressor(estimator=regressor, prefit=True) + scr_prefit.conformalize(X_conformalize, y_conformalize) + predictions_scr_prefit = scr_prefit.predict_interval(X_test) + + scr_not_prefit = SplitConformalRegressor(estimator=LinearRegression(), prefit=False) + scr_not_prefit.fit(X_train, y_train).conformalize(X_conformalize, y_conformalize) + predictions_scr_not_prefit = scr_not_prefit.predict_interval(X_test) + np.testing.assert_equal(predictions_scr_prefit, predictions_scr_not_prefit) + + +@pytest.mark.parametrize( + "split_technique,predict_method,dataset,estimator_class", + [ + ( + SplitConformalRegressor, + "predict_interval", + "dataset_regression", + DummyRegressor + ), + ( + ConformalizedQuantileRegressor, + "predict_interval", + "dataset_regression", + QuantileRegressor + ) + ] +) +class TestWrongMethodsOrderRaisesErrorForSplitTechniquesRegression: + def test_with_prefit_false( + self, + split_technique, + predict_method, + dataset, + estimator_class, + request + ): + dataset = request.getfixturevalue(dataset) + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset + estimator = estimator_class() + technique = split_technique(estimator=estimator, prefit=False) + + with pytest.raises(ValueError, match=r"call fit before calling conformalize"): + technique.conformalize( + X_conformalize, + y_conformalize + ) + + technique.fit(X_train, y_train) + + with pytest.raises(ValueError, match=r"fit method already called"): + technique.fit(X_train, y_train) + with pytest.raises( + ValueError, + match=r"call conformalize before calling predict" + ): + technique.predict(X_test) + + with pytest.raises( + ValueError, + match=f"call conformalize before calling {predict_method}" + ): + getattr(technique, predict_method)(X_test) + + technique.conformalize(X_conformalize, y_conformalize) + + with pytest.raises(ValueError, match=r"conformalize method already called"): + technique.conformalize(X_conformalize, y_conformalize) + + def test_with_prefit_true( + self, + split_technique, + predict_method, + dataset, + estimator_class, + request + ): + dataset = request.getfixturevalue(dataset) + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset + estimator = estimator_class() + estimator.fit(X_train, y_train) + + if split_technique == ConformalizedQuantileRegressor: + technique = split_technique(estimator=[estimator] * 3, prefit=True) + else: + technique = split_technique(estimator=estimator, prefit=True) + + with pytest.raises(ValueError, match=r"The fit method must be skipped"): + technique.fit(X_train, y_train) + with pytest.raises( + ValueError, + match=r"call conformalize before calling predict" + ): + technique.predict(X_test) + + with pytest.raises( + ValueError, + match=f"call conformalize before calling {predict_method}" + ): + getattr(technique, predict_method)(X_test) + + technique.conformalize(X_conformalize, y_conformalize) + + with pytest.raises(ValueError, match=r"conformalize method already called"): + technique.conformalize(X_conformalize, y_conformalize) + + +@pytest.mark.parametrize( + "cross_technique,predict_method,dataset,estimator_class", + [ + ( + CrossConformalRegressor, + "predict_interval", + "dataset_regression", + DummyRegressor + ), + ( + JackknifeAfterBootstrapRegressor, + "predict_interval", + "dataset_regression", + DummyRegressor + ) + ] +) +class TestWrongMethodsOrderRaisesErrorForCrossTechniques: + def test_wrong_methods_order( + self, + cross_technique, + predict_method, + dataset, + estimator_class, + request + ): + dataset = request.getfixturevalue(dataset) + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset + technique = cross_technique(estimator=estimator_class()) + + with pytest.raises( + ValueError, + match=r"call fit_conformalize before calling predict" + ): + technique.predict(X_test) + with pytest.raises( + ValueError, + match=f"call fit_conformalize before calling {predict_method}" + ): + getattr(technique, predict_method)(X_test) + + technique.fit_conformalize(X_conformalize, y_conformalize) + + with pytest.raises(ValueError, match=r"fit_conformalize method already called"): + technique.fit_conformalize(X_conformalize, y_conformalize) + + X_toy = np.array([0, 1, 2, 3, 4, 5]).reshape(-1, 1) y_toy = np.array([5, 7, 9, 11, 13, 15]) X, y = make_regression( @@ -1048,200 +1245,3 @@ def test_invalid_method(method: str) -> None: ValueError, match="(Invalid method.)|(Invalid conformity score.)*" ): mapie_estimator.fit(X_toy, y_toy) - - -class TestCheckAndConvertResamplingToCv: - def test_with_integer(self): - regressor = JackknifeAfterBootstrapRegressor() - cv = regressor._check_and_convert_resampling_to_cv(50) - - assert isinstance(cv, Subsample) - assert cv.n_resamplings == 50 - - def test_with_subsample(self): - custom_subsample = Subsample(n_resamplings=25, random_state=42) - regressor = JackknifeAfterBootstrapRegressor() - cv = regressor._check_and_convert_resampling_to_cv(custom_subsample) - - assert cv is custom_subsample - - def test_with_invalid_input(self): - regressor = JackknifeAfterBootstrapRegressor() - - with pytest.raises( - ValueError, - match="resampling must be an integer or a Subsample instance" - ): - regressor._check_and_convert_resampling_to_cv("invalid_input") - - -@pytest.fixture(scope="module") -def dataset_regression(): - X, y = make_regression( - n_samples=500, n_features=2, noise=1.0, random_state=random_state - ) - X_train, X_conf_test, y_train, y_conf_test = train_test_split( - X, y, random_state=random_state - ) - X_conformalize, X_test, y_conformalize, y_test = train_test_split( - X_conf_test, y_conf_test, random_state=random_state - ) - return X_train, X_conformalize, X_test, y_train, y_conformalize, y_test - - -def test_scr_same_predictions_prefit_not_prefit(dataset_regression) -> None: - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = ( - dataset_regression) - regressor = LinearRegression() - regressor.fit(X_train, y_train) - scr_prefit = SplitConformalRegressor(estimator=regressor, prefit=True) - scr_prefit.conformalize(X_conformalize, y_conformalize) - predictions_scr_prefit = scr_prefit.predict_interval(X_test) - - scr_not_prefit = SplitConformalRegressor(estimator=LinearRegression(), prefit=False) - scr_not_prefit.fit(X_train, y_train).conformalize(X_conformalize, y_conformalize) - predictions_scr_not_prefit = scr_not_prefit.predict_interval(X_test) - np.testing.assert_equal(predictions_scr_prefit, predictions_scr_not_prefit) - - -@pytest.mark.parametrize( - "split_technique,predict_method,dataset,estimator_class", - [ - ( - SplitConformalRegressor, - "predict_interval", - "dataset_regression", - DummyRegressor - ), - ( - ConformalizedQuantileRegressor, - "predict_interval", - "dataset_regression", - QuantileRegressor - ) - ] -) -class TestWrongMethodsOrderRaisesErrorForSplitTechniquesRegression: - def test_with_prefit_false( - self, - split_technique, - predict_method, - dataset, - estimator_class, - request - ): - dataset = request.getfixturevalue(dataset) - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset - estimator = estimator_class() - technique = split_technique(estimator=estimator, prefit=False) - - with pytest.raises(ValueError, match=r"call fit before calling conformalize"): - technique.conformalize( - X_conformalize, - y_conformalize - ) - - technique.fit(X_train, y_train) - - with pytest.raises(ValueError, match=r"fit method already called"): - technique.fit(X_train, y_train) - with pytest.raises( - ValueError, - match=r"call conformalize before calling predict" - ): - technique.predict(X_test) - - with pytest.raises( - ValueError, - match=f"call conformalize before calling {predict_method}" - ): - getattr(technique, predict_method)(X_test) - - technique.conformalize(X_conformalize, y_conformalize) - - with pytest.raises(ValueError, match=r"conformalize method already called"): - technique.conformalize(X_conformalize, y_conformalize) - - def test_with_prefit_true( - self, - split_technique, - predict_method, - dataset, - estimator_class, - request - ): - dataset = request.getfixturevalue(dataset) - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset - estimator = estimator_class() - estimator.fit(X_train, y_train) - - if split_technique == ConformalizedQuantileRegressor: - technique = split_technique(estimator=[estimator] * 3, prefit=True) - else: - technique = split_technique(estimator=estimator, prefit=True) - - with pytest.raises(ValueError, match=r"The fit method must be skipped"): - technique.fit(X_train, y_train) - with pytest.raises( - ValueError, - match=r"call conformalize before calling predict" - ): - technique.predict(X_test) - - with pytest.raises( - ValueError, - match=f"call conformalize before calling {predict_method}" - ): - getattr(technique, predict_method)(X_test) - - technique.conformalize(X_conformalize, y_conformalize) - - with pytest.raises(ValueError, match=r"conformalize method already called"): - technique.conformalize(X_conformalize, y_conformalize) - - -@pytest.mark.parametrize( - "cross_technique,predict_method,dataset,estimator_class", - [ - ( - CrossConformalRegressor, - "predict_interval", - "dataset_regression", - DummyRegressor - ), - ( - JackknifeAfterBootstrapRegressor, - "predict_interval", - "dataset_regression", - DummyRegressor - ) - ] -) -class TestWrongMethodsOrderRaisesErrorForCrossTechniques: - def test_wrong_methods_order( - self, - cross_technique, - predict_method, - dataset, - estimator_class, - request - ): - dataset = request.getfixturevalue(dataset) - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset - technique = cross_technique(estimator=estimator_class()) - - with pytest.raises( - ValueError, - match=r"call fit_conformalize before calling predict" - ): - technique.predict(X_test) - with pytest.raises( - ValueError, - match=f"call fit_conformalize before calling {predict_method}" - ): - getattr(technique, predict_method)(X_test) - - technique.fit_conformalize(X_conformalize, y_conformalize) - - with pytest.raises(ValueError, match=r"fit_conformalize method already called"): - technique.fit_conformalize(X_conformalize, y_conformalize) diff --git a/mapie/tests/test_utils.py b/mapie/tests/test_utils.py index 5a1d41f44..4c22ed42b 100644 --- a/mapie/tests/test_utils.py +++ b/mapie/tests/test_utils.py @@ -1,15 +1,13 @@ from __future__ import annotations -import inspect import logging import re -from typing import Any, Optional, Tuple, Union, Callable, Dict +from typing import Any, Optional, Tuple from unittest.mock import patch import numpy as np import pytest from numpy.random import RandomState -from sklearn.base import BaseEstimator, ClassifierMixin from sklearn.datasets import make_regression from sklearn.linear_model import LinearRegression from sklearn.model_selection import (BaseCrossValidator, KFold, LeaveOneOut, @@ -17,7 +15,6 @@ from sklearn.utils.validation import check_is_fitted from numpy.typing import ArrayLike, NDArray -from typing_extensions import Self from mapie.regression.quantile_regression import _MapieQuantileRegressor from mapie.utils import (_check_alpha, _check_alpha_and_n_samples, @@ -39,6 +36,250 @@ _raise_error_if_method_already_called, _raise_error_if_fit_called_in_prefit_mode) +@pytest.fixture(scope="module") +def dataset(): + X, y = make_regression( + n_samples=100, n_features=2, noise=1.0, random_state=random_state + ) + return X, y + + +class TestTrainConformalizeTestSplit: + + def test_error_sum_int_is_not_dataset_size(self, dataset): + X, y = dataset + with pytest.raises(ValueError): + train_conformalize_test_split( + X, y, train_size=1, conformalize_size=1, + test_size=1, random_state=random_state + ) + + def test_error_sum_float_is_not_1(self, dataset): + X, y = dataset + with pytest.raises(ValueError): + train_conformalize_test_split( + X, y, train_size=0.5, conformalize_size=0.5, + test_size=0.5, random_state=random_state + ) + + def test_error_sizes_are_int_and_float(self, dataset): + X, y = dataset + with pytest.raises(TypeError): + train_conformalize_test_split( + X, y, train_size=5, conformalize_size=0.5, + test_size=0.5, random_state=random_state + ) + + def test_3_floats(self, dataset): + X, y = dataset + ( + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test + ) = train_conformalize_test_split( + X, y, train_size=0.6, conformalize_size=0.2, + test_size=0.2, random_state=random_state + ) + assert len(X_train) == 60 + assert len(X_conformalize) == 20 + assert len(X_test) == 20 + + def test_3_ints(self, dataset): + X, y = dataset + ( + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test + ) = train_conformalize_test_split( + X, y, train_size=60, conformalize_size=20, + test_size=20, random_state=random_state + ) + assert len(X_train) == 60 + assert len(X_conformalize) == 20 + assert len(X_test) == 20 + + def test_random_state(self, dataset): + X, y = dataset + ( + X_train_1, X_conformalize_1, X_test_1, y_train_1, y_conformalize_1, y_test_1 + ) = train_conformalize_test_split( + X, y, train_size=60, conformalize_size=20, + test_size=20, random_state=random_state + ) + ( + X_train_2, X_conformalize_2, X_test_2, y_train_2, y_conformalize_2, y_test_2 + ) = train_conformalize_test_split( + X, y, train_size=60, conformalize_size=20, + test_size=20, random_state=random_state + ) + assert np.array_equal(X_train_1, X_train_2) + assert np.array_equal(X_conformalize_1, X_conformalize_2) + assert np.array_equal(X_test_1, X_test_2) + assert np.array_equal(y_train_1, y_train_2) + assert np.array_equal(y_conformalize_1, y_conformalize_2) + assert np.array_equal(y_test_1, y_test_2) + + def test_different_random_state(self, dataset): + X, y = dataset + ( + X_train_1, X_conformalize_1, X_test_1, y_train_1, y_conformalize_1, y_test_1 + ) = train_conformalize_test_split( + X, y, train_size=60, conformalize_size=20, + test_size=20, random_state=random_state + ) + ( + X_train_2, X_conformalize_2, X_test_2, y_train_2, y_conformalize_2, y_test_2 + ) = train_conformalize_test_split( + X, y, train_size=60, conformalize_size=20, + test_size=20, random_state=random_state + 1 + ) + assert not np.array_equal(X_train_1, X_train_2) + assert not np.array_equal(X_conformalize_1, X_conformalize_2) + assert not np.array_equal(X_test_1, X_test_2) + assert not np.array_equal(y_train_1, y_train_2) + assert not np.array_equal(y_conformalize_1, y_conformalize_2) + assert not np.array_equal(y_test_1, y_test_2) + + def test_shuffle_false(self, dataset): + X, y = dataset + ( + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test + ) = train_conformalize_test_split( + X, y, train_size=60, conformalize_size=20, + test_size=20, random_state=random_state, shuffle=False + ) + assert np.array_equal(np.concatenate((y_train, y_conformalize, y_test)), y) + + +@pytest.fixture +def point_predictions(): + return np.array([1, 2, 3]) + + +@pytest.fixture +def point_and_interval_predictions(): + return np.array([1, 2]), np.array([3, 4]) + + +@pytest.mark.parametrize( + "confidence_level, expected", + [ + (0.9, 0.1), + (0.7, 0.3), + (0.999, 0.001), + ] +) +def test_transform_confidence_level_to_alpha(confidence_level, expected): + result = _transform_confidence_level_to_alpha(confidence_level) + assert result == expected + assert str(result) == str(expected) # Ensure clean representation + + +class TestTransformConfidenceLevelToAlphaList: + def test_non_list_iterable(self): + confidence_level = (0.8, 0.7) # Testing a non-list iterable + assert _transform_confidence_level_to_alpha_list(confidence_level) == [0.2, 0.3] + + def test_transform_confidence_level_to_alpha_is_called(self): + with patch( + 'mapie.utils._transform_confidence_level_to_alpha' + ) as mock_transform_confidence_level_to_alpha: + _transform_confidence_level_to_alpha_list([0.2, 0.3]) + mock_transform_confidence_level_to_alpha.assert_called() + + +class TestCheckIfParamInAllowedValues: + def test_error(self): + with pytest.raises(ValueError): + _check_if_param_in_allowed_values("invalid_option", "", ["valid_option"]) + + def test_ok(self): + assert _check_if_param_in_allowed_values("valid", "", ["valid"]) is None + + +def test_check_cv_not_string(): + with pytest.raises(ValueError): + _check_cv_not_string("string") + + +class TestCastPointPredictionsToNdarray: + def test_error(self, point_and_interval_predictions): + with pytest.raises(TypeError): + _cast_point_predictions_to_ndarray(point_and_interval_predictions) + + def test_valid_ndarray(self, point_predictions): + point_predictions = np.array([1, 2, 3]) + result = _cast_point_predictions_to_ndarray(point_predictions) + assert result is point_predictions + assert isinstance(result, np.ndarray) + + +class TestCastPredictionsToNdarrayTuple: + def test_error(self, point_predictions): + with pytest.raises(TypeError): + _cast_predictions_to_ndarray_tuple(point_predictions) + + def test_valid_ndarray(self, point_and_interval_predictions): + result = _cast_predictions_to_ndarray_tuple(point_and_interval_predictions) + assert result is point_and_interval_predictions + assert isinstance(result, tuple) + assert isinstance(result[0], np.ndarray) + assert isinstance(result[1], np.ndarray) + + +@pytest.mark.parametrize( + "params, expected", [(None, {}), ({"a": 1, "b": 2}, {"a": 1, "b": 2})] +) +def test_prepare_params(params, expected): + assert _prepare_params(params) == expected + assert _prepare_params(params) is not params + + +class TestPrepareFitParamsAndSampleWeight: + def test_uses_prepare_params(self): + with patch('mapie.utils._prepare_params') as mock_prepare_params: + _prepare_fit_params_and_sample_weight({"param1": 1}) + mock_prepare_params.assert_called() + + def test_with_sample_weight(self): + fit_params = {"sample_weight": [0.1, 0.2, 0.3]} + assert _prepare_fit_params_and_sample_weight(fit_params) == ( + {}, + [0.1, 0.2, 0.3] + ) + + def test_without_sample_weight(self): + params = {"param1": 1} + assert _prepare_fit_params_and_sample_weight(params) == (params, None) + + +class TestRaiseErrorIfPreviousMethodNotCalled: + def test_raises_error_when_previous_method_not_called(self): + with pytest.raises(ValueError): + _raise_error_if_previous_method_not_called( + "current_method", "previous_method", False + ) + + def test_does_nothing_when_previous_method_called(self): + assert _raise_error_if_previous_method_not_called( + "current_method", "previous_method", True + ) is None + + +class TestRaiseErrorIfMethodAlreadyCalled: + def test_raises_error_when_method_already_called(self): + with pytest.raises(ValueError): + _raise_error_if_method_already_called("method", True) + + def test_does_nothing_when_method_not_called(self): + assert _raise_error_if_method_already_called("method", False) is None + + +class TestRaiseErrorIfFitCalledInPrefitMode: + def test_raises_error_in_prefit_mode(self): + with pytest.raises(ValueError): + _raise_error_if_fit_called_in_prefit_mode(True) + + def test_does_nothing_when_not_in_prefit_mode(self): + assert _raise_error_if_fit_called_in_prefit_mode(False) is None + + X_toy = np.array([0, 1, 2, 3, 4, 5]).reshape(-1, 1) y_toy = np.array([5, 7, 9, 11, 13, 15]) @@ -579,315 +820,3 @@ def test_invalid_n_samples_float(n_samples: float) -> None: ) ): _check_n_samples(X=X, n_samples=n_samples, indices=indices) - - -def train_test_split_shuffle( - X: NDArray, - y: NDArray, - test_size: float = None, - random_state: int = 42, - sample_weight: Optional[NDArray] = None, -) -> Union[Tuple[Any, Any, Any, Any], Tuple[Any, Any, Any, Any, Any, Any]]: - splitter = ShuffleSplit( - n_splits=1, - test_size=test_size, - random_state=random_state - ) - train_idx, test_idx = next(splitter.split(X)) - - X_train, X_test = X[train_idx], X[test_idx] - y_train, y_test = y[train_idx], y[test_idx] - if sample_weight is not None: - sample_weight_train = sample_weight[train_idx] - sample_weight_test = sample_weight[test_idx] - return X_train, X_test, y_train, y_test, sample_weight_train, sample_weight_test - - return X_train, X_test, y_train, y_test - - -def filter_params( - function: Callable, - params: Optional[Dict[str, Any]] = None -) -> Dict[str, Any]: - if params is None: - return {} - - model_params = inspect.signature(function).parameters - return {k: v for k, v in params.items() if k in model_params} - - -class DummyClassifierWithFitAndPredictParams(BaseEstimator, ClassifierMixin): - def __init__(self): - self.classes_ = None - self._dummy_fit_param = None - - def fit(self, X: ArrayLike, y: ArrayLike, dummy_fit_param: bool = False) -> Self: - self.classes_ = np.unique(y) - if len(self.classes_) < 2: - raise ValueError("Dummy classifier needs at least 3 classes") - self._dummy_fit_param = dummy_fit_param - return self - - def predict_proba(self, X: ArrayLike, dummy_predict_param: bool = False) -> NDArray: - probas = np.zeros((len(X), len(self.classes_))) - if self._dummy_fit_param & dummy_predict_param: - probas[:, 0] = 0.1 - probas[:, 1] = 0.9 - elif self._dummy_fit_param: - probas[:, 1] = 0.1 - probas[:, 2] = 0.9 - elif dummy_predict_param: - probas[:, 1] = 0.1 - probas[:, 0] = 0.9 - else: - probas[:, 2] = 0.1 - probas[:, 0] = 0.9 - return probas - - def predict(self, X: ArrayLike, dummy_predict_param: bool = False) -> NDArray: - y_preds_proba = self.predict_proba(X, dummy_predict_param) - return np.amax(y_preds_proba, axis=0) - - -@pytest.fixture(scope="module") -def dataset(): - X, y = make_regression( - n_samples=100, n_features=2, noise=1.0, random_state=random_state - ) - return X, y - - -class TestTrainConformalizeTestSplit: - - def test_error_sum_int_is_not_dataset_size(self, dataset): - X, y = dataset - with pytest.raises(ValueError): - train_conformalize_test_split( - X, y, train_size=1, conformalize_size=1, - test_size=1, random_state=random_state - ) - - def test_error_sum_float_is_not_1(self, dataset): - X, y = dataset - with pytest.raises(ValueError): - train_conformalize_test_split( - X, y, train_size=0.5, conformalize_size=0.5, - test_size=0.5, random_state=random_state - ) - - def test_error_sizes_are_int_and_float(self, dataset): - X, y = dataset - with pytest.raises(TypeError): - train_conformalize_test_split( - X, y, train_size=5, conformalize_size=0.5, - test_size=0.5, random_state=random_state - ) - - def test_3_floats(self, dataset): - X, y = dataset - ( - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test - ) = train_conformalize_test_split( - X, y, train_size=0.6, conformalize_size=0.2, - test_size=0.2, random_state=random_state - ) - assert len(X_train) == 60 - assert len(X_conformalize) == 20 - assert len(X_test) == 20 - - def test_3_ints(self, dataset): - X, y = dataset - ( - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test - ) = train_conformalize_test_split( - X, y, train_size=60, conformalize_size=20, - test_size=20, random_state=random_state - ) - assert len(X_train) == 60 - assert len(X_conformalize) == 20 - assert len(X_test) == 20 - - def test_random_state(self, dataset): - X, y = dataset - ( - X_train_1, X_conformalize_1, X_test_1, y_train_1, y_conformalize_1, y_test_1 - ) = train_conformalize_test_split( - X, y, train_size=60, conformalize_size=20, - test_size=20, random_state=random_state - ) - ( - X_train_2, X_conformalize_2, X_test_2, y_train_2, y_conformalize_2, y_test_2 - ) = train_conformalize_test_split( - X, y, train_size=60, conformalize_size=20, - test_size=20, random_state=random_state - ) - assert np.array_equal(X_train_1, X_train_2) - assert np.array_equal(X_conformalize_1, X_conformalize_2) - assert np.array_equal(X_test_1, X_test_2) - assert np.array_equal(y_train_1, y_train_2) - assert np.array_equal(y_conformalize_1, y_conformalize_2) - assert np.array_equal(y_test_1, y_test_2) - - def test_different_random_state(self, dataset): - X, y = dataset - ( - X_train_1, X_conformalize_1, X_test_1, y_train_1, y_conformalize_1, y_test_1 - ) = train_conformalize_test_split( - X, y, train_size=60, conformalize_size=20, - test_size=20, random_state=random_state - ) - ( - X_train_2, X_conformalize_2, X_test_2, y_train_2, y_conformalize_2, y_test_2 - ) = train_conformalize_test_split( - X, y, train_size=60, conformalize_size=20, - test_size=20, random_state=random_state + 1 - ) - assert not np.array_equal(X_train_1, X_train_2) - assert not np.array_equal(X_conformalize_1, X_conformalize_2) - assert not np.array_equal(X_test_1, X_test_2) - assert not np.array_equal(y_train_1, y_train_2) - assert not np.array_equal(y_conformalize_1, y_conformalize_2) - assert not np.array_equal(y_test_1, y_test_2) - - def test_shuffle_false(self, dataset): - X, y = dataset - ( - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test - ) = train_conformalize_test_split( - X, y, train_size=60, conformalize_size=20, - test_size=20, random_state=random_state, shuffle=False - ) - assert np.array_equal(np.concatenate((y_train, y_conformalize, y_test)), y) - - -@pytest.fixture -def point_predictions(): - return np.array([1, 2, 3]) - - -@pytest.fixture -def point_and_interval_predictions(): - return np.array([1, 2]), np.array([3, 4]) - - -@pytest.mark.parametrize( - "confidence_level, expected", - [ - (0.9, 0.1), - (0.7, 0.3), - (0.999, 0.001), - ] -) -def test_transform_confidence_level_to_alpha(confidence_level, expected): - result = _transform_confidence_level_to_alpha(confidence_level) - assert result == expected - assert str(result) == str(expected) # Ensure clean representation - - -class TestTransformConfidenceLevelToAlphaList: - def test_non_list_iterable(self): - confidence_level = (0.8, 0.7) # Testing a non-list iterable - assert _transform_confidence_level_to_alpha_list(confidence_level) == [0.2, 0.3] - - def test_transform_confidence_level_to_alpha_is_called(self): - with patch( - 'mapie.utils._transform_confidence_level_to_alpha' - ) as mock_transform_confidence_level_to_alpha: - _transform_confidence_level_to_alpha_list([0.2, 0.3]) - mock_transform_confidence_level_to_alpha.assert_called() - - -class TestCheckIfParamInAllowedValues: - def test_error(self): - with pytest.raises(ValueError): - _check_if_param_in_allowed_values("invalid_option", "", ["valid_option"]) - - def test_ok(self): - assert _check_if_param_in_allowed_values("valid", "", ["valid"]) is None - - -def test_check_cv_not_string(): - with pytest.raises(ValueError): - _check_cv_not_string("string") - - -class TestCastPointPredictionsToNdarray: - def test_error(self, point_and_interval_predictions): - with pytest.raises(TypeError): - _cast_point_predictions_to_ndarray(point_and_interval_predictions) - - def test_valid_ndarray(self, point_predictions): - point_predictions = np.array([1, 2, 3]) - result = _cast_point_predictions_to_ndarray(point_predictions) - assert result is point_predictions - assert isinstance(result, np.ndarray) - - -class TestCastPredictionsToNdarrayTuple: - def test_error(self, point_predictions): - with pytest.raises(TypeError): - _cast_predictions_to_ndarray_tuple(point_predictions) - - def test_valid_ndarray(self, point_and_interval_predictions): - result = _cast_predictions_to_ndarray_tuple(point_and_interval_predictions) - assert result is point_and_interval_predictions - assert isinstance(result, tuple) - assert isinstance(result[0], np.ndarray) - assert isinstance(result[1], np.ndarray) - - -@pytest.mark.parametrize( - "params, expected", [(None, {}), ({"a": 1, "b": 2}, {"a": 1, "b": 2})] -) -def test_prepare_params(params, expected): - assert _prepare_params(params) == expected - assert _prepare_params(params) is not params - - -class TestPrepareFitParamsAndSampleWeight: - def test_uses_prepare_params(self): - with patch('mapie.utils._prepare_params') as mock_prepare_params: - _prepare_fit_params_and_sample_weight({"param1": 1}) - mock_prepare_params.assert_called() - - def test_with_sample_weight(self): - fit_params = {"sample_weight": [0.1, 0.2, 0.3]} - assert _prepare_fit_params_and_sample_weight(fit_params) == ( - {}, - [0.1, 0.2, 0.3] - ) - - def test_without_sample_weight(self): - params = {"param1": 1} - assert _prepare_fit_params_and_sample_weight(params) == (params, None) - - -class TestRaiseErrorIfPreviousMethodNotCalled: - def test_raises_error_when_previous_method_not_called(self): - with pytest.raises(ValueError): - _raise_error_if_previous_method_not_called( - "current_method", "previous_method", False - ) - - def test_does_nothing_when_previous_method_called(self): - assert _raise_error_if_previous_method_not_called( - "current_method", "previous_method", True - ) is None - - -class TestRaiseErrorIfMethodAlreadyCalled: - def test_raises_error_when_method_already_called(self): - with pytest.raises(ValueError): - _raise_error_if_method_already_called("method", True) - - def test_does_nothing_when_method_not_called(self): - assert _raise_error_if_method_already_called("method", False) is None - - -class TestRaiseErrorIfFitCalledInPrefitMode: - def test_raises_error_in_prefit_mode(self): - with pytest.raises(ValueError): - _raise_error_if_fit_called_in_prefit_mode(True) - - def test_does_nothing_when_not_in_prefit_mode(self): - assert _raise_error_if_fit_called_in_prefit_mode(False) is None From 1e64e11d91d6dc62d5abe2aa48a21ee81715143f Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Fri, 1 Aug 2025 11:45:17 +0200 Subject: [PATCH 04/14] Merge tests v0 and v1 refacto --- mapie/tests/test_classification.py | 145 +--------- mapie/tests/test_common.py | 285 ++++++++++++++------ mapie/tests/test_conformity_scores_utils.py | 1 + mapie/tests/test_non_regression_v0_to_v1.py | 78 +++++- mapie/tests/test_regression.py | 178 +----------- mapie/tests/test_utils.py | 1 + 6 files changed, 290 insertions(+), 398 deletions(-) diff --git a/mapie/tests/test_classification.py b/mapie/tests/test_classification.py index 9f96460ca..905e7a4e2 100644 --- a/mapie/tests/test_classification.py +++ b/mapie/tests/test_classification.py @@ -24,8 +24,7 @@ from typing_extensions import TypedDict from numpy.typing import ArrayLike, NDArray -from mapie.classification import _MapieClassifier, SplitConformalClassifier, \ - CrossConformalClassifier +from mapie.classification import _MapieClassifier from mapie.conformity_scores import ( LACConformityScore, RAPSConformityScore, @@ -37,148 +36,6 @@ from mapie.utils import check_proba_normalized from mapie.metrics.classification import classification_coverage_score -@pytest.fixture(scope="module") -def dataset_classification(): - X, y = make_classification( - n_samples=500, n_informative=5, n_classes=4, random_state=random_state, - ) - X_train, X_conf_test, y_train, y_conf_test = train_test_split( - X, y, random_state=random_state - ) - X_conformalize, X_test, y_conformalize, y_test = train_test_split( - X_conf_test, y_conf_test, random_state=random_state - ) - return X_train, X_conformalize, X_test, y_train, y_conformalize, y_test - - -@pytest.mark.parametrize( - "split_technique,predict_method,dataset,estimator_class", - [ - ( - SplitConformalClassifier, - "predict_set", - "dataset_classification", - DummyClassifier - ) - ] -) -class TestWrongMethodsOrderRaisesErrorForSplitTechniquesClassification: - def test_with_prefit_false( - self, - split_technique, - predict_method, - dataset, - estimator_class, - request - ): - dataset = request.getfixturevalue(dataset) - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset - estimator = estimator_class() - technique = split_technique(estimator=estimator, prefit=False) - - with pytest.raises(ValueError, match=r"call fit before calling conformalize"): - technique.conformalize( - X_conformalize, - y_conformalize - ) - - technique.fit(X_train, y_train) - - with pytest.raises(ValueError, match=r"fit method already called"): - technique.fit(X_train, y_train) - with pytest.raises( - ValueError, - match=r"call conformalize before calling predict" - ): - technique.predict(X_test) - - with pytest.raises( - ValueError, - match=f"call conformalize before calling {predict_method}" - ): - getattr(technique, predict_method)(X_test) - - technique.conformalize(X_conformalize, y_conformalize) - - with pytest.raises(ValueError, match=r"conformalize method already called"): - technique.conformalize(X_conformalize, y_conformalize) - - def test_with_prefit_true( - self, - split_technique, - predict_method, - dataset, - estimator_class, - request - ): - dataset = request.getfixturevalue(dataset) - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset - estimator = estimator_class() - estimator.fit(X_train, y_train) - - technique = split_technique(estimator=estimator, prefit=True) - - with pytest.raises(ValueError, match=r"The fit method must be skipped"): - technique.fit(X_train, y_train) - with pytest.raises( - ValueError, - match=r"call conformalize before calling predict" - ): - technique.predict(X_test) - - with pytest.raises( - ValueError, - match=f"call conformalize before calling {predict_method}" - ): - getattr(technique, predict_method)(X_test) - - technique.conformalize(X_conformalize, y_conformalize) - - with pytest.raises(ValueError, match=r"conformalize method already called"): - technique.conformalize(X_conformalize, y_conformalize) - - -@pytest.mark.parametrize( - "cross_technique,predict_method,dataset,estimator_class", - [ - ( - CrossConformalClassifier, - "predict_set", - "dataset_classification", - DummyClassifier - ) - ] -) -class TestWrongMethodsOrderRaisesErrorForCrossTechniques: - def test_wrong_methods_order( - self, - cross_technique, - predict_method, - dataset, - estimator_class, - request - ): - dataset = request.getfixturevalue(dataset) - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset - technique = cross_technique(estimator=estimator_class()) - - with pytest.raises( - ValueError, - match=r"call fit_conformalize before calling predict" - ): - technique.predict(X_test) - with pytest.raises( - ValueError, - match=f"call fit_conformalize before calling {predict_method}" - ): - getattr(technique, predict_method)(X_test) - - technique.fit_conformalize(X_conformalize, y_conformalize) - - with pytest.raises(ValueError, match=r"fit_conformalize method already called"): - technique.fit_conformalize(X_conformalize, y_conformalize) - - random_state = 42 WRONG_INCLUDE_LABELS = ["randomised", "True", "False", "other", 1, 2.5, (1, 2)] diff --git a/mapie/tests/test_common.py b/mapie/tests/test_common.py index 126926699..701086416 100644 --- a/mapie/tests/test_common.py +++ b/mapie/tests/test_common.py @@ -1,88 +1,223 @@ -import inspect from inspect import signature -from typing import Any, List, Tuple, Optional, Union, Callable, Dict +from typing import Any, List, Tuple import numpy as np import pytest -from numpy._typing import NDArray, ArrayLike -from sklearn.base import BaseEstimator, ClassifierMixin +from sklearn.base import BaseEstimator +from sklearn.datasets import make_regression, make_classification +from sklearn.dummy import DummyRegressor, DummyClassifier from sklearn.exceptions import NotFittedError -from sklearn.linear_model import LinearRegression, LogisticRegression -from sklearn.model_selection import KFold, ShuffleSplit +from sklearn.linear_model import LinearRegression, LogisticRegression, QuantileRegressor +from sklearn.model_selection import KFold, train_test_split from sklearn.pipeline import make_pipeline from sklearn.utils.validation import check_is_fitted -from typing_extensions import Self - -from mapie.classification import _MapieClassifier -from mapie.regression.regression import _MapieRegressor -from mapie.regression.quantile_regression import _MapieQuantileRegressor - -def train_test_split_shuffle( - X: NDArray, - y: NDArray, - test_size: Optional[float] = None, - random_state: int = 42, - sample_weight: Optional[NDArray] = None, -) -> Union[Tuple[Any, Any, Any, Any], Tuple[Any, Any, Any, Any, Any, Any]]: - splitter = ShuffleSplit( - n_splits=1, - test_size=test_size, - random_state=random_state + +from mapie.classification import _MapieClassifier, SplitConformalClassifier, \ + CrossConformalClassifier +from mapie.regression.regression import _MapieRegressor, SplitConformalRegressor, \ + CrossConformalRegressor, JackknifeAfterBootstrapRegressor +from mapie.regression.quantile_regression import _MapieQuantileRegressor, \ + ConformalizedQuantileRegressor + +RANDOM_STATE = 1 + + +@pytest.fixture(scope="module") +def dataset_regression(): + X, y = make_regression( + n_samples=500, n_features=2, noise=1.0, random_state=RANDOM_STATE + ) + X_train, X_conf_test, y_train, y_conf_test = train_test_split( + X, y, random_state=RANDOM_STATE + ) + X_conformalize, X_test, y_conformalize, y_test = train_test_split( + X_conf_test, y_conf_test, random_state=RANDOM_STATE + ) + return X_train, X_conformalize, X_test, y_train, y_conformalize, y_test + + +@pytest.fixture(scope="module") +def dataset_classification(): + X, y = make_classification( + n_samples=500, n_informative=5, n_classes=4, random_state=RANDOM_STATE, + ) + X_train, X_conf_test, y_train, y_conf_test = train_test_split( + X, y, random_state=RANDOM_STATE + ) + X_conformalize, X_test, y_conformalize, y_test = train_test_split( + X_conf_test, y_conf_test, random_state=RANDOM_STATE ) - train_idx, test_idx = next(splitter.split(X)) - - X_train, X_test = X[train_idx], X[test_idx] - y_train, y_test = y[train_idx], y[test_idx] - if sample_weight is not None: - sample_weight_train = sample_weight[train_idx] - sample_weight_test = sample_weight[test_idx] - return X_train, X_test, y_train, y_test, sample_weight_train, sample_weight_test - - return X_train, X_test, y_train, y_test - - -def filter_params( - function: Callable, - params: Optional[Dict[str, Any]] = None -) -> Dict[str, Any]: - if params is None: - return {} - - model_params = inspect.signature(function).parameters - return {k: v for k, v in params.items() if k in model_params} - - -class DummyClassifierWithFitAndPredictParams(BaseEstimator, ClassifierMixin): - def __init__(self): - self.classes_ = None - self._dummy_fit_param = None - - def fit(self, X: ArrayLike, y: ArrayLike, dummy_fit_param: bool = False) -> Self: - self.classes_ = np.unique(y) - if len(self.classes_) < 2: - raise ValueError("Dummy classifier needs at least 3 classes") - self._dummy_fit_param = dummy_fit_param - return self - - def predict_proba(self, X: ArrayLike, dummy_predict_param: bool = False) -> NDArray: - probas = np.zeros((len(X), len(self.classes_))) - if self._dummy_fit_param & dummy_predict_param: - probas[:, 0] = 0.1 - probas[:, 1] = 0.9 - elif self._dummy_fit_param: - probas[:, 1] = 0.1 - probas[:, 2] = 0.9 - elif dummy_predict_param: - probas[:, 1] = 0.1 - probas[:, 0] = 0.9 + return X_train, X_conformalize, X_test, y_train, y_conformalize, y_test + + +def test_scr_same_predictions_prefit_not_prefit(dataset_regression) -> None: + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = ( + dataset_regression) + regressor = LinearRegression() + regressor.fit(X_train, y_train) + scr_prefit = SplitConformalRegressor(estimator=regressor, prefit=True) + scr_prefit.conformalize(X_conformalize, y_conformalize) + predictions_scr_prefit = scr_prefit.predict_interval(X_test) + + scr_not_prefit = SplitConformalRegressor(estimator=LinearRegression(), prefit=False) + scr_not_prefit.fit(X_train, y_train).conformalize(X_conformalize, y_conformalize) + predictions_scr_not_prefit = scr_not_prefit.predict_interval(X_test) + np.testing.assert_equal(predictions_scr_prefit, predictions_scr_not_prefit) + + +@pytest.mark.parametrize( + "split_technique,predict_method,dataset,estimator_class", + [ + ( + SplitConformalRegressor, + "predict_interval", + "dataset_regression", + DummyRegressor + ), + ( + ConformalizedQuantileRegressor, + "predict_interval", + "dataset_regression", + QuantileRegressor + ), + ( + SplitConformalClassifier, + "predict_set", + "dataset_classification", + DummyClassifier + ) + ] +) +class TestWrongMethodsOrderRaisesErrorForSplitTechniques: + def test_with_prefit_false( + self, + split_technique, + predict_method, + dataset, + estimator_class, + request + ): + dataset = request.getfixturevalue(dataset) + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset + estimator = estimator_class() + technique = split_technique(estimator=estimator, prefit=False) + + with pytest.raises(ValueError, match=r"call fit before calling conformalize"): + technique.conformalize( + X_conformalize, + y_conformalize + ) + + technique.fit(X_train, y_train) + + with pytest.raises(ValueError, match=r"fit method already called"): + technique.fit(X_train, y_train) + with pytest.raises( + ValueError, + match=r"call conformalize before calling predict" + ): + technique.predict(X_test) + + with pytest.raises( + ValueError, + match=f"call conformalize before calling {predict_method}" + ): + getattr(technique, predict_method)(X_test) + + technique.conformalize(X_conformalize, y_conformalize) + + with pytest.raises(ValueError, match=r"conformalize method already called"): + technique.conformalize(X_conformalize, y_conformalize) + + def test_with_prefit_true( + self, + split_technique, + predict_method, + dataset, + estimator_class, + request + ): + dataset = request.getfixturevalue(dataset) + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset + estimator = estimator_class() + estimator.fit(X_train, y_train) + + if split_technique == ConformalizedQuantileRegressor: + technique = split_technique(estimator=[estimator] * 3, prefit=True) else: - probas[:, 2] = 0.1 - probas[:, 0] = 0.9 - return probas + technique = split_technique(estimator=estimator, prefit=True) + + with pytest.raises(ValueError, match=r"The fit method must be skipped"): + technique.fit(X_train, y_train) + with pytest.raises( + ValueError, + match=r"call conformalize before calling predict" + ): + technique.predict(X_test) - def predict(self, X: ArrayLike, dummy_predict_param: bool = False) -> NDArray: - y_preds_proba = self.predict_proba(X, dummy_predict_param) - return np.amax(y_preds_proba, axis=0) + with pytest.raises( + ValueError, + match=f"call conformalize before calling {predict_method}" + ): + getattr(technique, predict_method)(X_test) + + technique.conformalize(X_conformalize, y_conformalize) + + with pytest.raises(ValueError, match=r"conformalize method already called"): + technique.conformalize(X_conformalize, y_conformalize) + + +@pytest.mark.parametrize( + "cross_technique,predict_method,dataset,estimator_class", + [ + ( + CrossConformalRegressor, + "predict_interval", + "dataset_regression", + DummyRegressor + ), + ( + JackknifeAfterBootstrapRegressor, + "predict_interval", + "dataset_regression", + DummyRegressor + ), + ( + CrossConformalClassifier, + "predict_set", + "dataset_classification", + DummyClassifier + ), + ] +) +class TestWrongMethodsOrderRaisesErrorForCrossTechniques: + def test_wrong_methods_order( + self, + cross_technique, + predict_method, + dataset, + estimator_class, + request + ): + dataset = request.getfixturevalue(dataset) + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset + technique = cross_technique(estimator=estimator_class()) + + with pytest.raises( + ValueError, + match=r"call fit_conformalize before calling predict" + ): + technique.predict(X_test) + with pytest.raises( + ValueError, + match=f"call fit_conformalize before calling {predict_method}" + ): + getattr(technique, predict_method)(X_test) + + technique.fit_conformalize(X_conformalize, y_conformalize) + + with pytest.raises(ValueError, match=r"fit_conformalize method already called"): + technique.fit_conformalize(X_conformalize, y_conformalize) X_toy = np.arange(18).reshape(-1, 1) diff --git a/mapie/tests/test_conformity_scores_utils.py b/mapie/tests/test_conformity_scores_utils.py index 2e7bf342d..5a1a55790 100644 --- a/mapie/tests/test_conformity_scores_utils.py +++ b/mapie/tests/test_conformity_scores_utils.py @@ -11,6 +11,7 @@ from mapie.conformity_scores.utils import check_and_select_conformity_score + class TestCheckAndSelectConformityScore: @pytest.mark.parametrize( diff --git a/mapie/tests/test_non_regression_v0_to_v1.py b/mapie/tests/test_non_regression_v0_to_v1.py index bea385fa8..86fe323f8 100644 --- a/mapie/tests/test_non_regression_v0_to_v1.py +++ b/mapie/tests/test_non_regression_v0_to_v1.py @@ -1,14 +1,18 @@ -from typing import Type, Union, Dict, Optional +import inspect +from typing import Type, Union, Dict, Optional, Callable, Any, Tuple import numpy as np import pytest from numpy._typing import ArrayLike, NDArray from numpy.random import RandomState +from sklearn.base import BaseEstimator, ClassifierMixin from sklearn.compose import TransformedTargetRegressor from sklearn.datasets import make_classification, make_regression from sklearn.ensemble import RandomForestClassifier, GradientBoostingRegressor from sklearn.linear_model import LogisticRegression, LinearRegression, QuantileRegressor -from sklearn.model_selection import LeaveOneOut, GroupKFold, train_test_split +from sklearn.model_selection import LeaveOneOut, GroupKFold, train_test_split, \ + ShuffleSplit +from typing_extensions import Self from mapie.classification import _MapieClassifier, SplitConformalClassifier, \ CrossConformalClassifier @@ -20,8 +24,6 @@ ConformalizedQuantileRegressor from mapie.regression.regression import _MapieRegressor, SplitConformalRegressor from mapie.subsample import Subsample -from mapie.tests.test_utils import train_test_split_shuffle, \ - DummyClassifierWithFitAndPredictParams, filter_params RANDOM_STATE = 1 K_FOLDS = 3 @@ -1036,3 +1038,71 @@ def compare_model_predictions_and_intervals( # condition to remove when optimize_beta works # keep assertion assert v1_pred_intervals.shape == (len(X_conf), 2, n_alpha) + + +def train_test_split_shuffle( + X: NDArray, + y: NDArray, + test_size: Optional[float] = None, + random_state: int = 42, + sample_weight: Optional[NDArray] = None, +) -> Union[Tuple[Any, Any, Any, Any], Tuple[Any, Any, Any, Any, Any, Any]]: + splitter = ShuffleSplit( + n_splits=1, + test_size=test_size, + random_state=random_state + ) + train_idx, test_idx = next(splitter.split(X)) + + X_train, X_test = X[train_idx], X[test_idx] + y_train, y_test = y[train_idx], y[test_idx] + if sample_weight is not None: + sample_weight_train = sample_weight[train_idx] + sample_weight_test = sample_weight[test_idx] + return X_train, X_test, y_train, y_test, sample_weight_train, sample_weight_test + + return X_train, X_test, y_train, y_test + + +def filter_params( + function: Callable, + params: Optional[Dict[str, Any]] = None +) -> Dict[str, Any]: + if params is None: + return {} + + model_params = inspect.signature(function).parameters + return {k: v for k, v in params.items() if k in model_params} + + +class DummyClassifierWithFitAndPredictParams(BaseEstimator, ClassifierMixin): + def __init__(self): + self.classes_ = None + self._dummy_fit_param = None + + def fit(self, X: ArrayLike, y: ArrayLike, dummy_fit_param: bool = False) -> Self: + self.classes_ = np.unique(y) + if len(self.classes_) < 2: + raise ValueError("Dummy classifier needs at least 3 classes") + self._dummy_fit_param = dummy_fit_param + return self + + def predict_proba(self, X: ArrayLike, dummy_predict_param: bool = False) -> NDArray: + probas = np.zeros((len(X), len(self.classes_))) + if self._dummy_fit_param & dummy_predict_param: + probas[:, 0] = 0.1 + probas[:, 1] = 0.9 + elif self._dummy_fit_param: + probas[:, 1] = 0.1 + probas[:, 2] = 0.9 + elif dummy_predict_param: + probas[:, 1] = 0.1 + probas[:, 0] = 0.9 + else: + probas[:, 2] = 0.1 + probas[:, 0] = 0.9 + return probas + + def predict(self, X: ArrayLike, dummy_predict_param: bool = False) -> NDArray: + y_preds_proba = self.predict_proba(X, dummy_predict_param) + return np.amax(y_preds_proba, axis=0) diff --git a/mapie/tests/test_regression.py b/mapie/tests/test_regression.py index f99cd4358..8a207afaf 100644 --- a/mapie/tests/test_regression.py +++ b/mapie/tests/test_regression.py @@ -13,7 +13,7 @@ from sklearn.dummy import DummyRegressor from sklearn.ensemble import GradientBoostingRegressor from sklearn.impute import SimpleImputer -from sklearn.linear_model import LinearRegression, QuantileRegressor +from sklearn.linear_model import LinearRegression from sklearn.model_selection import ( GroupKFold, KFold, LeaveOneOut, PredefinedSplit, ShuffleSplit, train_test_split, LeaveOneGroupOut, LeavePGroupsOut @@ -33,11 +33,11 @@ from mapie.metrics.regression import ( regression_coverage_score, ) -from mapie.regression import ConformalizedQuantileRegressor from mapie.regression.regression import _MapieRegressor, \ - JackknifeAfterBootstrapRegressor, SplitConformalRegressor, CrossConformalRegressor + JackknifeAfterBootstrapRegressor from mapie.subsample import Subsample + class TestCheckAndConvertResamplingToCv: def test_with_integer(self): regressor = JackknifeAfterBootstrapRegressor() @@ -63,178 +63,6 @@ def test_with_invalid_input(self): regressor._check_and_convert_resampling_to_cv("invalid_input") -@pytest.fixture(scope="module") -def dataset_regression(): - X, y = make_regression( - n_samples=500, n_features=2, noise=1.0, random_state=random_state - ) - X_train, X_conf_test, y_train, y_conf_test = train_test_split( - X, y, random_state=random_state - ) - X_conformalize, X_test, y_conformalize, y_test = train_test_split( - X_conf_test, y_conf_test, random_state=random_state - ) - return X_train, X_conformalize, X_test, y_train, y_conformalize, y_test - - -def test_scr_same_predictions_prefit_not_prefit(dataset_regression) -> None: - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = ( - dataset_regression) - regressor = LinearRegression() - regressor.fit(X_train, y_train) - scr_prefit = SplitConformalRegressor(estimator=regressor, prefit=True) - scr_prefit.conformalize(X_conformalize, y_conformalize) - predictions_scr_prefit = scr_prefit.predict_interval(X_test) - - scr_not_prefit = SplitConformalRegressor(estimator=LinearRegression(), prefit=False) - scr_not_prefit.fit(X_train, y_train).conformalize(X_conformalize, y_conformalize) - predictions_scr_not_prefit = scr_not_prefit.predict_interval(X_test) - np.testing.assert_equal(predictions_scr_prefit, predictions_scr_not_prefit) - - -@pytest.mark.parametrize( - "split_technique,predict_method,dataset,estimator_class", - [ - ( - SplitConformalRegressor, - "predict_interval", - "dataset_regression", - DummyRegressor - ), - ( - ConformalizedQuantileRegressor, - "predict_interval", - "dataset_regression", - QuantileRegressor - ) - ] -) -class TestWrongMethodsOrderRaisesErrorForSplitTechniquesRegression: - def test_with_prefit_false( - self, - split_technique, - predict_method, - dataset, - estimator_class, - request - ): - dataset = request.getfixturevalue(dataset) - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset - estimator = estimator_class() - technique = split_technique(estimator=estimator, prefit=False) - - with pytest.raises(ValueError, match=r"call fit before calling conformalize"): - technique.conformalize( - X_conformalize, - y_conformalize - ) - - technique.fit(X_train, y_train) - - with pytest.raises(ValueError, match=r"fit method already called"): - technique.fit(X_train, y_train) - with pytest.raises( - ValueError, - match=r"call conformalize before calling predict" - ): - technique.predict(X_test) - - with pytest.raises( - ValueError, - match=f"call conformalize before calling {predict_method}" - ): - getattr(technique, predict_method)(X_test) - - technique.conformalize(X_conformalize, y_conformalize) - - with pytest.raises(ValueError, match=r"conformalize method already called"): - technique.conformalize(X_conformalize, y_conformalize) - - def test_with_prefit_true( - self, - split_technique, - predict_method, - dataset, - estimator_class, - request - ): - dataset = request.getfixturevalue(dataset) - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset - estimator = estimator_class() - estimator.fit(X_train, y_train) - - if split_technique == ConformalizedQuantileRegressor: - technique = split_technique(estimator=[estimator] * 3, prefit=True) - else: - technique = split_technique(estimator=estimator, prefit=True) - - with pytest.raises(ValueError, match=r"The fit method must be skipped"): - technique.fit(X_train, y_train) - with pytest.raises( - ValueError, - match=r"call conformalize before calling predict" - ): - technique.predict(X_test) - - with pytest.raises( - ValueError, - match=f"call conformalize before calling {predict_method}" - ): - getattr(technique, predict_method)(X_test) - - technique.conformalize(X_conformalize, y_conformalize) - - with pytest.raises(ValueError, match=r"conformalize method already called"): - technique.conformalize(X_conformalize, y_conformalize) - - -@pytest.mark.parametrize( - "cross_technique,predict_method,dataset,estimator_class", - [ - ( - CrossConformalRegressor, - "predict_interval", - "dataset_regression", - DummyRegressor - ), - ( - JackknifeAfterBootstrapRegressor, - "predict_interval", - "dataset_regression", - DummyRegressor - ) - ] -) -class TestWrongMethodsOrderRaisesErrorForCrossTechniques: - def test_wrong_methods_order( - self, - cross_technique, - predict_method, - dataset, - estimator_class, - request - ): - dataset = request.getfixturevalue(dataset) - X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset - technique = cross_technique(estimator=estimator_class()) - - with pytest.raises( - ValueError, - match=r"call fit_conformalize before calling predict" - ): - technique.predict(X_test) - with pytest.raises( - ValueError, - match=f"call fit_conformalize before calling {predict_method}" - ): - getattr(technique, predict_method)(X_test) - - technique.fit_conformalize(X_conformalize, y_conformalize) - - with pytest.raises(ValueError, match=r"fit_conformalize method already called"): - technique.fit_conformalize(X_conformalize, y_conformalize) - - X_toy = np.array([0, 1, 2, 3, 4, 5]).reshape(-1, 1) y_toy = np.array([5, 7, 9, 11, 13, 15]) X, y = make_regression( diff --git a/mapie/tests/test_utils.py b/mapie/tests/test_utils.py index 4c22ed42b..b3c3b30f3 100644 --- a/mapie/tests/test_utils.py +++ b/mapie/tests/test_utils.py @@ -36,6 +36,7 @@ _raise_error_if_method_already_called, _raise_error_if_fit_called_in_prefit_mode) + @pytest.fixture(scope="module") def dataset(): X, y = make_regression( From 758b73f96193b987dcdbd6308572897f64188404 Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Fri, 1 Aug 2025 15:25:57 +0200 Subject: [PATCH 05/14] Merge tests v0 and v1 refacto --- mapie/tests/test_non_regression_v0_to_v1.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mapie/tests/test_non_regression_v0_to_v1.py b/mapie/tests/test_non_regression_v0_to_v1.py index 86fe323f8..f237be6e7 100644 --- a/mapie/tests/test_non_regression_v0_to_v1.py +++ b/mapie/tests/test_non_regression_v0_to_v1.py @@ -205,7 +205,7 @@ def params_split_test_5(): "params_split_test_5", ] ) -def test_split_classification(dataset, params_, request): +def test_split(dataset, params_, request): X, y, X_train, X_conformalize, y_train, y_conformalize = ( dataset["X"], dataset["y"], @@ -379,7 +379,7 @@ def params_cross_test_4(): "params_cross_test_4", ] ) -def test_cross_classification(dataset, params_, request): +def test_cross(dataset, params_, request): X, y = dataset["X"], dataset["y"] params = extract_params(request.getfixturevalue(params_)) @@ -710,7 +710,7 @@ def run_v1_pipeline_cross_or_jackknife(params): "params", params_test_cases_cross + params_test_cases_jackknife ) -def test_cross_and_jackknife_regression(params: dict) -> None: +def test_cross_and_jackknife(params: dict) -> None: v0_preds, v0_pred_intervals = run_v0_pipeline_cross_or_jackknife(params) ( v1_preds, @@ -816,7 +816,7 @@ def test_cross_and_jackknife_regression(params: dict) -> None: @pytest.mark.parametrize("params_split", params_test_cases_split) -def test_intervals_and_predictions_exact_equality_split_regression( +def test_intervals_and_predictions_exact_equality_split( params_split: dict) -> None: v0_params = params_split["v0"] v1_params = params_split["v1"] @@ -932,7 +932,7 @@ def test_intervals_and_predictions_exact_equality_split_regression( @pytest.mark.parametrize("params_quantile", params_test_cases_quantile) -def test_intervals_and_predictions_exact_equality_quantile_regression( +def test_intervals_and_predictions_exact_equality_quantile( params_quantile: dict ) -> None: v0_params = params_quantile["v0"] From 9a22e2be70d7c8569327bf4474d21ac61949379f Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Fri, 1 Aug 2025 17:53:43 +0200 Subject: [PATCH 06/14] Merge tests v0 and v1: fix typing error --- mapie/tests/test_non_regression_v0_to_v1.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/mapie/tests/test_non_regression_v0_to_v1.py b/mapie/tests/test_non_regression_v0_to_v1.py index f237be6e7..7f25c6eee 100644 --- a/mapie/tests/test_non_regression_v0_to_v1.py +++ b/mapie/tests/test_non_regression_v0_to_v1.py @@ -979,7 +979,14 @@ def compare_model_predictions_and_intervals( n_alpha = 1 if test_size is not None: - X_train, X_conf, y_train, y_conf = train_test_split_shuffle( + ( + X_train, + X_conf, + y_train, + y_conf, + sample_weight_train, + sample_weight_conf, + ) = train_test_split_shuffle( X, y, test_size=test_size, @@ -1046,7 +1053,7 @@ def train_test_split_shuffle( test_size: Optional[float] = None, random_state: int = 42, sample_weight: Optional[NDArray] = None, -) -> Union[Tuple[Any, Any, Any, Any], Tuple[Any, Any, Any, Any, Any, Any]]: +) -> Tuple[Any, Any, Any, Any, Any, Any]: splitter = ShuffleSplit( n_splits=1, test_size=test_size, @@ -1059,9 +1066,11 @@ def train_test_split_shuffle( if sample_weight is not None: sample_weight_train = sample_weight[train_idx] sample_weight_test = sample_weight[test_idx] - return X_train, X_test, y_train, y_test, sample_weight_train, sample_weight_test + else: + sample_weight_train = None + sample_weight_test = None - return X_train, X_test, y_train, y_test + return X_train, X_test, y_train, y_test, sample_weight_train, sample_weight_test def filter_params( From 63b9d02a5178b18c00e6fe73f0846a5a08c05be4 Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Fri, 1 Aug 2025 18:30:35 +0200 Subject: [PATCH 07/14] Merge tests v0 and v1: fix typing error --- mapie/tests/test_non_regression_v0_to_v1.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mapie/tests/test_non_regression_v0_to_v1.py b/mapie/tests/test_non_regression_v0_to_v1.py index 7f25c6eee..fec0dd304 100644 --- a/mapie/tests/test_non_regression_v0_to_v1.py +++ b/mapie/tests/test_non_regression_v0_to_v1.py @@ -1096,7 +1096,7 @@ def fit(self, X: ArrayLike, y: ArrayLike, dummy_fit_param: bool = False) -> Self self._dummy_fit_param = dummy_fit_param return self - def predict_proba(self, X: ArrayLike, dummy_predict_param: bool = False) -> NDArray: + def predict_proba(self, X: NDArray, dummy_predict_param: bool = False) -> NDArray: probas = np.zeros((len(X), len(self.classes_))) if self._dummy_fit_param & dummy_predict_param: probas[:, 0] = 0.1 @@ -1112,6 +1112,6 @@ def predict_proba(self, X: ArrayLike, dummy_predict_param: bool = False) -> NDAr probas[:, 0] = 0.9 return probas - def predict(self, X: ArrayLike, dummy_predict_param: bool = False) -> NDArray: + def predict(self, X: NDArray, dummy_predict_param: bool = False) -> NDArray: y_preds_proba = self.predict_proba(X, dummy_predict_param) return np.amax(y_preds_proba, axis=0) From 5deee9812d29e4482100d45a3184b1ba05ee4a78 Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Mon, 18 Aug 2025 11:04:45 +0200 Subject: [PATCH 08/14] Merge tests v0 and v1: fix typing error --- mapie/tests/test_non_regression_v0_to_v1.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/mapie/tests/test_non_regression_v0_to_v1.py b/mapie/tests/test_non_regression_v0_to_v1.py index fec0dd304..5134b7b6b 100644 --- a/mapie/tests/test_non_regression_v0_to_v1.py +++ b/mapie/tests/test_non_regression_v0_to_v1.py @@ -3,6 +3,7 @@ import numpy as np import pytest +from _pytest.fixtures import FixtureRequest from numpy._typing import ArrayLike, NDArray from numpy.random import RandomState from sklearn.base import BaseEstimator, ClassifierMixin @@ -205,7 +206,11 @@ def params_split_test_5(): "params_split_test_5", ] ) -def test_split(dataset, params_, request): +def test_split( + dataset: Dict[str, Any], + params_: str, + request: FixtureRequest +) -> None: X, y, X_train, X_conformalize, y_train, y_conformalize = ( dataset["X"], dataset["y"], @@ -236,7 +241,7 @@ def test_split(dataset, params_, request): v0_preds, v0_pred_sets = v0.predict(X_conformalize, **params["v0_predict"]) v1_preds, v1_pred_sets = v1.predict_set(X_conformalize, **params["v1_predict_set"]) - v1_preds_using_predict: ArrayLike = v1.predict(X_conformalize) + v1_preds_using_predict: NDArray = v1.predict(X_conformalize) np.testing.assert_array_equal(v0_preds, v1_preds) np.testing.assert_array_equal(v0_pred_sets, v1_pred_sets) @@ -379,7 +384,11 @@ def params_cross_test_4(): "params_cross_test_4", ] ) -def test_cross(dataset, params_, request): +def test_cross( + dataset: Dict[str, Any], + params_: str, + request: FixtureRequest +): X, y = dataset["X"], dataset["y"] params = extract_params(request.getfixturevalue(params_)) @@ -393,7 +402,7 @@ def test_cross(dataset, params_, request): v0_preds, v0_pred_sets = v0.predict(X, **params["v0_predict"]) v1_preds, v1_pred_sets = v1.predict_set(X, **params["v1_predict_set"]) - v1_preds_using_predict: ArrayLike = v1.predict(X) + v1_preds_using_predict: NDArray = v1.predict(X) np.testing.assert_array_equal(v0_preds, v1_preds) np.testing.assert_array_equal(v0_pred_sets, v1_pred_sets) @@ -1089,7 +1098,7 @@ def __init__(self): self.classes_ = None self._dummy_fit_param = None - def fit(self, X: ArrayLike, y: ArrayLike, dummy_fit_param: bool = False) -> Self: + def fit(self, X: NDArray, y: NDArray, dummy_fit_param: bool = False) -> Self: self.classes_ = np.unique(y) if len(self.classes_) < 2: raise ValueError("Dummy classifier needs at least 3 classes") From 9c5e678fc0a2c46625fe40f3d267f764f204000b Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Mon, 18 Aug 2025 12:05:59 +0200 Subject: [PATCH 09/14] Merge tests v0 and v1: fix typing error --- mapie/tests/test_non_regression_v0_to_v1.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/mapie/tests/test_non_regression_v0_to_v1.py b/mapie/tests/test_non_regression_v0_to_v1.py index 5134b7b6b..14f620c22 100644 --- a/mapie/tests/test_non_regression_v0_to_v1.py +++ b/mapie/tests/test_non_regression_v0_to_v1.py @@ -833,7 +833,7 @@ def test_intervals_and_predictions_exact_equality_split( test_size = v1_params.get("test_size", None) prefit = v1_params.get("prefit", False) - compare_model_predictions_and_intervals( + compare_model_predictions_and_intervals_split_and_quantile( model_v0=_MapieRegressor, model_v1=SplitConformalRegressor, X=X, @@ -950,7 +950,7 @@ def test_intervals_and_predictions_exact_equality_quantile( test_size = v1_params.get("test_size", None) prefit = v1_params.get("prefit", False) - compare_model_predictions_and_intervals( + compare_model_predictions_and_intervals_split_and_quantile( model_v0=_MapieQuantileRegressor, model_v1=ConformalizedQuantileRegressor, X=X, @@ -963,12 +963,10 @@ def test_intervals_and_predictions_exact_equality_quantile( ) -def compare_model_predictions_and_intervals( +def compare_model_predictions_and_intervals_split_and_quantile( model_v0: Type[_MapieRegressor], model_v1: Type[Union[ SplitConformalRegressor, - CrossConformalRegressor, - JackknifeAfterBootstrapRegressor, ConformalizedQuantileRegressor ]], X: NDArray, From 3a60c4587759cbcca51e2fad33bb2cd1ed79194a Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Mon, 18 Aug 2025 14:20:59 +0200 Subject: [PATCH 10/14] Merge tests v0 and v1: fix coverage --- mapie/tests/test_non_regression_v0_to_v1.py | 60 +++++++-------------- 1 file changed, 18 insertions(+), 42 deletions(-) diff --git a/mapie/tests/test_non_regression_v0_to_v1.py b/mapie/tests/test_non_regression_v0_to_v1.py index 14f620c22..cfe978d1b 100644 --- a/mapie/tests/test_non_regression_v0_to_v1.py +++ b/mapie/tests/test_non_regression_v0_to_v1.py @@ -977,30 +977,24 @@ def compare_model_predictions_and_intervals_split_and_quantile( test_size: Optional[float] = None, random_state: int = RANDOM_STATE, ) -> None: - if v0_params.get("alpha"): - if isinstance(v0_params["alpha"], float): - n_alpha = 1 - else: - n_alpha = len(v0_params["alpha"]) - else: + if isinstance(v0_params["alpha"], float): n_alpha = 1 - - if test_size is not None: - ( - X_train, - X_conf, - y_train, - y_conf, - sample_weight_train, - sample_weight_conf, - ) = train_test_split_shuffle( - X, - y, - test_size=test_size, - random_state=random_state, - ) else: - X_train, X_conf, y_train, y_conf = X, X, y, y + n_alpha = len(v0_params["alpha"]) + + ( + X_train, + X_conf, + y_train, + y_conf, + sample_weight_train, + sample_weight_conf, + ) = train_test_split_shuffle( + X, + y, + test_size=test_size, + random_state=random_state, + ) if prefit: estimator = v0_params["estimator"] @@ -1084,8 +1078,6 @@ def filter_params( function: Callable, params: Optional[Dict[str, Any]] = None ) -> Dict[str, Any]: - if params is None: - return {} model_params = inspect.signature(function).parameters return {k: v for k, v in params.items() if k in model_params} @@ -1098,27 +1090,11 @@ def __init__(self): def fit(self, X: NDArray, y: NDArray, dummy_fit_param: bool = False) -> Self: self.classes_ = np.unique(y) - if len(self.classes_) < 2: - raise ValueError("Dummy classifier needs at least 3 classes") self._dummy_fit_param = dummy_fit_param return self def predict_proba(self, X: NDArray, dummy_predict_param: bool = False) -> NDArray: probas = np.zeros((len(X), len(self.classes_))) - if self._dummy_fit_param & dummy_predict_param: - probas[:, 0] = 0.1 - probas[:, 1] = 0.9 - elif self._dummy_fit_param: - probas[:, 1] = 0.1 - probas[:, 2] = 0.9 - elif dummy_predict_param: - probas[:, 1] = 0.1 - probas[:, 0] = 0.9 - else: - probas[:, 2] = 0.1 - probas[:, 0] = 0.9 + probas[:, 0] = 0.1 + probas[:, 1] = 0.9 return probas - - def predict(self, X: NDArray, dummy_predict_param: bool = False) -> NDArray: - y_preds_proba = self.predict_proba(X, dummy_predict_param) - return np.amax(y_preds_proba, axis=0) From 3fcb75114dac2dc37c0229afa4fcb83a352417c7 Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Mon, 18 Aug 2025 15:33:55 +0200 Subject: [PATCH 11/14] Merge tests v0 and v1: fix typing --- mapie/tests/test_non_regression_v0_to_v1.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mapie/tests/test_non_regression_v0_to_v1.py b/mapie/tests/test_non_regression_v0_to_v1.py index cfe978d1b..3c749c069 100644 --- a/mapie/tests/test_non_regression_v0_to_v1.py +++ b/mapie/tests/test_non_regression_v0_to_v1.py @@ -1076,9 +1076,8 @@ def train_test_split_shuffle( def filter_params( function: Callable, - params: Optional[Dict[str, Any]] = None + params: Dict[str, Any] ) -> Dict[str, Any]: - model_params = inspect.signature(function).parameters return {k: v for k, v in params.items() if k in model_params} From bec76d95d3cf16b99a4d8abe54c8e2429489a70d Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Tue, 19 Aug 2025 12:17:30 +0200 Subject: [PATCH 12/14] Update HISTORY.rst --- HISTORY.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.rst b/HISTORY.rst index 1ef748509..cadf56f74 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,6 +5,7 @@ History 1.x.x (2025-xx-xx) ------------------ +* Merge tests from MAPIE v1 with v0 legacy tests * Fix double inference when using `predict_set` function in split conformal classification * Add FAQ entry in the documentation about ongoing works to extend MAPIE for LLM control * MAPIE now supports Python versions up to the latest release (currently 3.13) From 11906d3e91d69b4dad9bfb75c55f2e09ca8cb508 Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Tue, 19 Aug 2025 13:55:26 +0200 Subject: [PATCH 13/14] Update HISTORY.rst --- HISTORY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index cadf56f74..94b6a6671 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,7 +5,7 @@ History 1.x.x (2025-xx-xx) ------------------ -* Merge tests from MAPIE v1 with v0 legacy tests +* Merge MAPIE v1 tests with v0 legacy tests * Fix double inference when using `predict_set` function in split conformal classification * Add FAQ entry in the documentation about ongoing works to extend MAPIE for LLM control * MAPIE now supports Python versions up to the latest release (currently 3.13) From 9a07e3cb4dfae15dbbd8df5d08720cec36257a93 Mon Sep 17 00:00:00 2001 From: FaustinPulveric Date: Tue, 26 Aug 2025 10:41:51 +0200 Subject: [PATCH 14/14] Remove entry in HISTORY.rst --- HISTORY.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 94b6a6671..1ef748509 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,7 +5,6 @@ History 1.x.x (2025-xx-xx) ------------------ -* Merge MAPIE v1 tests with v0 legacy tests * Fix double inference when using `predict_set` function in split conformal classification * Add FAQ entry in the documentation about ongoing works to extend MAPIE for LLM control * MAPIE now supports Python versions up to the latest release (currently 3.13)