Skip to content

Commit a158dce

Browse files
FaustinPulvericFaustinPulveric
authored
Merge tests folders v0 & v1 (refacto still needed) (#737)
1 parent b27e0ba commit a158dce

14 files changed

+1021
-1108
lines changed

‎Makefile

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
### Checks that are run in GitHub CI ###
66
lint:
7-
flake8 examples mapie notebooks tests_v1 --max-line-length=88
7+
flake8 examples mapie notebooks --max-line-length=88
88

99
type-check:
1010
mypy mapie
@@ -14,7 +14,7 @@ coverage:
1414
--cov-branch \
1515
--cov=mapie \
1616
--cov-report term-missing \
17-
--pyargs mapie tests_v1 \
17+
--pyargs mapie \
1818
--cov-fail-under=100 \
1919
--no-cov-on-fail \
2020
--doctest-modules
@@ -37,7 +37,6 @@ all-checks:
3737

3838
tests:
3939
pytest -vs --doctest-modules mapie
40-
python -m pytest -vs tests_v1
4140

4241
clean-doc:
4342
$(MAKE) clean -C doc

‎mapie/tests/test_common.py

Lines changed: 211 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,221 @@
44
import numpy as np
55
import pytest
66
from sklearn.base import BaseEstimator
7+
from sklearn.datasets import make_regression, make_classification
8+
from sklearn.dummy import DummyRegressor, DummyClassifier
79
from sklearn.exceptions import NotFittedError
8-
from sklearn.linear_model import LinearRegression, LogisticRegression
9-
from sklearn.model_selection import KFold
10+
from sklearn.linear_model import LinearRegression, LogisticRegression, QuantileRegressor
11+
from sklearn.model_selection import KFold, train_test_split
1012
from sklearn.pipeline import make_pipeline
1113
from sklearn.utils.validation import check_is_fitted
1214

13-
from mapie.classification import _MapieClassifier
14-
from mapie.regression.regression import _MapieRegressor
15-
from mapie.regression.quantile_regression import _MapieQuantileRegressor
15+
from mapie.classification import _MapieClassifier, SplitConformalClassifier, \
16+
CrossConformalClassifier
17+
from mapie.regression.regression import _MapieRegressor, SplitConformalRegressor, \
18+
CrossConformalRegressor, JackknifeAfterBootstrapRegressor
19+
from mapie.regression.quantile_regression import _MapieQuantileRegressor, \
20+
ConformalizedQuantileRegressor
21+
22+
RANDOM_STATE = 1
23+
24+
25+
@pytest.fixture(scope="module")
26+
def dataset_regression():
27+
X, y = make_regression(
28+
n_samples=500, n_features=2, noise=1.0, random_state=RANDOM_STATE
29+
)
30+
X_train, X_conf_test, y_train, y_conf_test = train_test_split(
31+
X, y, random_state=RANDOM_STATE
32+
)
33+
X_conformalize, X_test, y_conformalize, y_test = train_test_split(
34+
X_conf_test, y_conf_test, random_state=RANDOM_STATE
35+
)
36+
return X_train, X_conformalize, X_test, y_train, y_conformalize, y_test
37+
38+
39+
@pytest.fixture(scope="module")
40+
def dataset_classification():
41+
X, y = make_classification(
42+
n_samples=500, n_informative=5, n_classes=4, random_state=RANDOM_STATE,
43+
)
44+
X_train, X_conf_test, y_train, y_conf_test = train_test_split(
45+
X, y, random_state=RANDOM_STATE
46+
)
47+
X_conformalize, X_test, y_conformalize, y_test = train_test_split(
48+
X_conf_test, y_conf_test, random_state=RANDOM_STATE
49+
)
50+
return X_train, X_conformalize, X_test, y_train, y_conformalize, y_test
51+
52+
53+
def test_scr_same_predictions_prefit_not_prefit(dataset_regression) -> None:
54+
X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = (
55+
dataset_regression)
56+
regressor = LinearRegression()
57+
regressor.fit(X_train, y_train)
58+
scr_prefit = SplitConformalRegressor(estimator=regressor, prefit=True)
59+
scr_prefit.conformalize(X_conformalize, y_conformalize)
60+
predictions_scr_prefit = scr_prefit.predict_interval(X_test)
61+
62+
scr_not_prefit = SplitConformalRegressor(estimator=LinearRegression(), prefit=False)
63+
scr_not_prefit.fit(X_train, y_train).conformalize(X_conformalize, y_conformalize)
64+
predictions_scr_not_prefit = scr_not_prefit.predict_interval(X_test)
65+
np.testing.assert_equal(predictions_scr_prefit, predictions_scr_not_prefit)
66+
67+
68+
@pytest.mark.parametrize(
69+
"split_technique,predict_method,dataset,estimator_class",
70+
[
71+
(
72+
SplitConformalRegressor,
73+
"predict_interval",
74+
"dataset_regression",
75+
DummyRegressor
76+
),
77+
(
78+
ConformalizedQuantileRegressor,
79+
"predict_interval",
80+
"dataset_regression",
81+
QuantileRegressor
82+
),
83+
(
84+
SplitConformalClassifier,
85+
"predict_set",
86+
"dataset_classification",
87+
DummyClassifier
88+
)
89+
]
90+
)
91+
class TestWrongMethodsOrderRaisesErrorForSplitTechniques:
92+
def test_with_prefit_false(
93+
self,
94+
split_technique,
95+
predict_method,
96+
dataset,
97+
estimator_class,
98+
request
99+
):
100+
dataset = request.getfixturevalue(dataset)
101+
X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset
102+
estimator = estimator_class()
103+
technique = split_technique(estimator=estimator, prefit=False)
104+
105+
with pytest.raises(ValueError, match=r"call fit before calling conformalize"):
106+
technique.conformalize(
107+
X_conformalize,
108+
y_conformalize
109+
)
110+
111+
technique.fit(X_train, y_train)
112+
113+
with pytest.raises(ValueError, match=r"fit method already called"):
114+
technique.fit(X_train, y_train)
115+
with pytest.raises(
116+
ValueError,
117+
match=r"call conformalize before calling predict"
118+
):
119+
technique.predict(X_test)
120+
121+
with pytest.raises(
122+
ValueError,
123+
match=f"call conformalize before calling {predict_method}"
124+
):
125+
getattr(technique, predict_method)(X_test)
126+
127+
technique.conformalize(X_conformalize, y_conformalize)
128+
129+
with pytest.raises(ValueError, match=r"conformalize method already called"):
130+
technique.conformalize(X_conformalize, y_conformalize)
131+
132+
def test_with_prefit_true(
133+
self,
134+
split_technique,
135+
predict_method,
136+
dataset,
137+
estimator_class,
138+
request
139+
):
140+
dataset = request.getfixturevalue(dataset)
141+
X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset
142+
estimator = estimator_class()
143+
estimator.fit(X_train, y_train)
144+
145+
if split_technique == ConformalizedQuantileRegressor:
146+
technique = split_technique(estimator=[estimator] * 3, prefit=True)
147+
else:
148+
technique = split_technique(estimator=estimator, prefit=True)
149+
150+
with pytest.raises(ValueError, match=r"The fit method must be skipped"):
151+
technique.fit(X_train, y_train)
152+
with pytest.raises(
153+
ValueError,
154+
match=r"call conformalize before calling predict"
155+
):
156+
technique.predict(X_test)
157+
158+
with pytest.raises(
159+
ValueError,
160+
match=f"call conformalize before calling {predict_method}"
161+
):
162+
getattr(technique, predict_method)(X_test)
163+
164+
technique.conformalize(X_conformalize, y_conformalize)
165+
166+
with pytest.raises(ValueError, match=r"conformalize method already called"):
167+
technique.conformalize(X_conformalize, y_conformalize)
168+
169+
170+
@pytest.mark.parametrize(
171+
"cross_technique,predict_method,dataset,estimator_class",
172+
[
173+
(
174+
CrossConformalRegressor,
175+
"predict_interval",
176+
"dataset_regression",
177+
DummyRegressor
178+
),
179+
(
180+
JackknifeAfterBootstrapRegressor,
181+
"predict_interval",
182+
"dataset_regression",
183+
DummyRegressor
184+
),
185+
(
186+
CrossConformalClassifier,
187+
"predict_set",
188+
"dataset_classification",
189+
DummyClassifier
190+
),
191+
]
192+
)
193+
class TestWrongMethodsOrderRaisesErrorForCrossTechniques:
194+
def test_wrong_methods_order(
195+
self,
196+
cross_technique,
197+
predict_method,
198+
dataset,
199+
estimator_class,
200+
request
201+
):
202+
dataset = request.getfixturevalue(dataset)
203+
X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset
204+
technique = cross_technique(estimator=estimator_class())
205+
206+
with pytest.raises(
207+
ValueError,
208+
match=r"call fit_conformalize before calling predict"
209+
):
210+
technique.predict(X_test)
211+
with pytest.raises(
212+
ValueError,
213+
match=f"call fit_conformalize before calling {predict_method}"
214+
):
215+
getattr(technique, predict_method)(X_test)
216+
217+
technique.fit_conformalize(X_conformalize, y_conformalize)
218+
219+
with pytest.raises(ValueError, match=r"fit_conformalize method already called"):
220+
technique.fit_conformalize(X_conformalize, y_conformalize)
221+
16222

17223
X_toy = np.arange(18).reshape(-1, 1)
18224
y_toy = np.array(

‎mapie/tests/test_conformity_scores_utils.py

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,37 @@
33
import numpy as np
44
import pytest
55

6+
from mapie.conformity_scores import AbsoluteConformityScore, BaseRegressionScore, \
7+
GammaConformityScore, LACConformityScore, BaseClassificationScore, \
8+
TopKConformityScore
69
from mapie.conformity_scores.sets.utils import get_true_label_position
710
from numpy.typing import NDArray
811

12+
from mapie.conformity_scores.utils import check_and_select_conformity_score
13+
14+
15+
class TestCheckAndSelectConformityScore:
16+
17+
@pytest.mark.parametrize(
18+
"score, score_type, expected_class", [
19+
(AbsoluteConformityScore(), BaseRegressionScore, AbsoluteConformityScore),
20+
("gamma", BaseRegressionScore, GammaConformityScore),
21+
(LACConformityScore(), BaseClassificationScore, LACConformityScore),
22+
("top_k", BaseClassificationScore, TopKConformityScore),
23+
]
24+
)
25+
def test_with_valid_inputs(self, score, score_type, expected_class):
26+
result = check_and_select_conformity_score(score, score_type)
27+
assert isinstance(result, expected_class)
28+
29+
@pytest.mark.parametrize(
30+
"score_type", [BaseRegressionScore, BaseClassificationScore]
31+
)
32+
def test_with_invalid_input(self, score_type):
33+
with pytest.raises(ValueError):
34+
check_and_select_conformity_score("I'm not a valid input :(", score_type)
35+
36+
937
Y_TRUE_PROBA_PLACE = [
1038
[
1139
np.array([2, 0]),

0 commit comments

Comments
 (0)