Skip to content

Commit f60eae7

Browse files
voorhsvoorhs
committed
add pipeline tests for each scoring module
1 parent 778d413 commit f60eae7

16 files changed

+374
-1
lines changed

‎tests/modules/scoring/test_bert.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import numpy as np
66
import pytest
77

8+
from autointent import Pipeline
89
from autointent.context.data_handler import DataHandler
910
from autointent.modules import BertScorer
1011

@@ -115,3 +116,26 @@ def test_bert_cache_clearing(dataset):
115116
# Should raise exception after clearing cache
116117
with pytest.raises(RuntimeError):
117118
scorer.predict(test_data)
119+
120+
121+
def test_bert_in_pipeline(dataset):
122+
"""Test BertScorer as part of an AutoML pipeline."""
123+
search_space = [
124+
{
125+
"node_type": "scoring",
126+
"search_space": [
127+
{
128+
"module_name": "bert",
129+
"classification_model_config": [{"model_name": "prajjwal1/bert-tiny"}],
130+
"num_train_epochs": [1],
131+
"batch_size": [8],
132+
}
133+
],
134+
},
135+
{"node_type": "decision", "search_space": [{"module_name": "argmax"}]},
136+
]
137+
138+
pipeline = Pipeline.from_search_space(search_space)
139+
pipeline.fit(dataset)
140+
predictions = pipeline.predict(["test utterance"])
141+
assert len(predictions) == 1

‎tests/modules/scoring/test_catboost.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import numpy as np
66
import pytest
77

8+
from autointent import Pipeline
89
from autointent.context.data_handler import DataHandler
910
from autointent.modules import CatBoostScorer
1011
from tests.conftest import get_test_embedder_config
@@ -146,3 +147,26 @@ def test_catboost_cache_clearing(dataset):
146147
scorer.clear_cache()
147148
with pytest.raises(RuntimeError):
148149
scorer.predict(test_data)
150+
151+
152+
def test_catboost_in_pipeline(dataset):
153+
"""Test CatBoostScorer as part of an AutoML pipeline."""
154+
search_space = [
155+
{
156+
"node_type": "scoring",
157+
"search_space": [
158+
{
159+
"module_name": "catboost",
160+
"iterations": [50],
161+
"learning_rate": [0.05],
162+
"features_type": ["embedding"],
163+
}
164+
],
165+
},
166+
{"node_type": "decision", "search_space": [{"module_name": "argmax"}]},
167+
]
168+
169+
pipeline = Pipeline.from_search_space(search_space)
170+
pipeline.fit(dataset)
171+
predictions = pipeline.predict(["test utterance"])
172+
assert len(predictions) == 1

‎tests/modules/scoring/test_cnn.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import numpy as np
66
import pytest
77

8+
from autointent import Pipeline
89
from autointent.configs import VocabConfig
910
from autointent.context.data_handler import DataHandler
1011
from autointent.modules.scoring import CNNScorer
@@ -120,3 +121,25 @@ def test_cnn_scorer_dump_load(dataset):
120121
finally:
121122
# Clean up
122123
shutil.rmtree(temp_dir_path, ignore_errors=True) # workaround for windows permission error
124+
125+
126+
def test_cnn_in_pipeline(dataset):
127+
"""Test CNNScorer as part of an AutoML pipeline."""
128+
search_space = [
129+
{
130+
"node_type": "scoring",
131+
"search_space": [
132+
{
133+
"module_name": "cnn",
134+
"embed_dim": [8],
135+
"num_train_epochs": [1],
136+
}
137+
],
138+
},
139+
{"node_type": "decision", "search_space": [{"module_name": "argmax"}]},
140+
]
141+
142+
pipeline = Pipeline.from_search_space(search_space)
143+
pipeline.fit(dataset)
144+
predictions = pipeline.predict(["test utterance"])
145+
assert len(predictions) == 1

‎tests/modules/scoring/test_description_bi.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import numpy as np
44
import pytest
55

6+
from autointent import Pipeline
67
from autointent.context.data_handler import DataHandler
78
from autointent.modules import BiEncoderDescriptionScorer
89

@@ -56,3 +57,25 @@ def test_description_scorer(dataset, expected_prediction, multilabel):
5657
new_scorer = BiEncoderDescriptionScorer.load(temp_dir)
5758
new_predictions = new_scorer.predict(test_utterances)
5859
np.testing.assert_almost_equal(predictions, new_predictions, decimal=5)
60+
61+
62+
def test_description_bi_in_pipeline(dataset):
63+
"""Test BiEncoderDescriptionScorer as part of an AutoML pipeline."""
64+
search_space = [
65+
{
66+
"node_type": "scoring",
67+
"search_space": [
68+
{
69+
"module_name": "description_bi",
70+
"embedder_config": [{"model_name": "sergeyzh/rubert-tiny-turbo"}],
71+
"temperature": [0.3],
72+
}
73+
],
74+
},
75+
{"node_type": "decision", "search_space": [{"module_name": "argmax"}]},
76+
]
77+
78+
pipeline = Pipeline.from_search_space(search_space)
79+
pipeline.fit(dataset)
80+
predictions = pipeline.predict(["test utterance"])
81+
assert len(predictions) == 1

‎tests/modules/scoring/test_description_cross.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import numpy as np
44
import pytest
55

6+
from autointent import Pipeline
67
from autointent.context.data_handler import DataHandler
78
from autointent.modules import CrossEncoderDescriptionScorer
89

@@ -64,3 +65,25 @@ def test_description_scorer_cross_encoder(dataset, expected_prediction, multilab
6465
np.testing.assert_almost_equal(predictions, loaded_predictions, decimal=5)
6566

6667
new_scorer.clear_cache()
68+
69+
70+
def test_description_cross_in_pipeline(dataset):
71+
"""Test CrossEncoderDescriptionScorer as part of an AutoML pipeline."""
72+
search_space = [
73+
{
74+
"node_type": "scoring",
75+
"search_space": [
76+
{
77+
"module_name": "description_cross",
78+
"cross_encoder_config": [{"model_name": "cross-encoder/ms-marco-MiniLM-L6-v2"}],
79+
"temperature": [0.3],
80+
}
81+
],
82+
},
83+
{"node_type": "decision", "search_space": [{"module_name": "argmax"}]},
84+
]
85+
86+
pipeline = Pipeline.from_search_space(search_space)
87+
pipeline.fit(dataset)
88+
predictions = pipeline.predict(["test utterance"])
89+
assert len(predictions) == 1

‎tests/modules/scoring/test_description_llm.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import numpy as np
55
import pytest
66

7+
from autointent import Pipeline
78
from autointent.context.data_handler import DataHandler
89
from autointent.modules import LLMDescriptionScorer
910

@@ -50,3 +51,28 @@ def test_description_scorer_llm(dataset, multilabel):
5051
new_scorer = LLMDescriptionScorer.load(temp_dir)
5152
new_predictions = new_scorer.predict(test_utterances)
5253
np.testing.assert_almost_equal(predictions, new_predictions, decimal=5)
54+
55+
56+
@pytest.mark.skipif(
57+
not os.getenv("OPENAI_API_KEY") or not os.getenv("OPENAI_MODEL_NAME"),
58+
reason="OPENAI_API_KEY and OPENAI_MODEL_NAME environment variables are required for this test",
59+
)
60+
def test_llm_description_in_pipeline(dataset):
61+
"""Test LLMDescriptionScorer as part of an AutoML pipeline."""
62+
search_space = [
63+
{
64+
"node_type": "scoring",
65+
"search_space": [
66+
{
67+
"module_name": "description_llm",
68+
"temperature": [0.3],
69+
}
70+
],
71+
},
72+
{"node_type": "decision", "search_space": [{"module_name": "argmax"}]},
73+
]
74+
75+
pipeline = Pipeline.from_search_space(search_space)
76+
pipeline.fit(dataset)
77+
predictions = pipeline.predict(["test utterance"])
78+
assert len(predictions) == 1

‎tests/modules/scoring/test_dnnc.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import numpy as np
44
import pytest
55

6+
from autointent import Pipeline
67
from autointent.context.data_handler import DataHandler
78
from autointent.modules import DNNCScorer
89

@@ -41,3 +42,25 @@ def test_base_dnnc(dataset, train_head, pred_score):
4142
new_scorer = DNNCScorer.load(temp_dir)
4243
new_predictions = new_scorer.predict(test_data)
4344
np.testing.assert_almost_equal(predictions, new_predictions, decimal=5)
45+
46+
47+
def test_dnnc_in_pipeline(dataset):
48+
"""Test DNNCScorer as part of an AutoML pipeline."""
49+
search_space = [
50+
{
51+
"node_type": "scoring",
52+
"search_space": [
53+
{
54+
"module_name": "dnnc",
55+
"cross_encoder_config": [{"model_name": "cross-encoder/ms-marco-MiniLM-L6-v2"}],
56+
"k": [3],
57+
}
58+
],
59+
},
60+
{"node_type": "decision", "search_space": [{"module_name": "argmax"}]},
61+
]
62+
63+
pipeline = Pipeline.from_search_space(search_space)
64+
pipeline.fit(dataset)
65+
predictions = pipeline.predict(["test utterance"])
66+
assert len(predictions) == 1

‎tests/modules/scoring/test_gcn_scorer.py‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
import pytest
33
import torch
44

5-
from autointent import Dataset
5+
from autointent import Dataset, Pipeline
66
from autointent.modules.scoring import GCNScorer
77
from tests.conftest import get_test_embedder_config
88

@@ -91,3 +91,25 @@ def test_gcn_scorer_dump_load(tmp_path, multilabel_dataset):
9191
loaded_predictions = loaded_scorer.predict(test_utterances)
9292

9393
np.testing.assert_allclose(original_predictions, loaded_predictions, atol=1e-6)
94+
95+
96+
def test_gcn_in_pipeline(dataset):
97+
"""Test GCNScorer as part of an AutoML pipeline."""
98+
search_space = [
99+
{
100+
"node_type": "scoring",
101+
"search_space": [
102+
{
103+
"module_name": "gcn",
104+
"num_train_epochs": [1],
105+
"batch_size": [8],
106+
}
107+
],
108+
},
109+
{"node_type": "decision", "search_space": [{"module_name": "threshold", "thresh": [0.5]}]},
110+
]
111+
112+
pipeline = Pipeline.from_search_space(search_space)
113+
pipeline.fit(dataset.to_multilabel())
114+
predictions = pipeline.predict(["test utterance"])
115+
assert len(predictions) == 1

‎tests/modules/scoring/test_knn.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import numpy as np
44

5+
from autointent import Pipeline
56
from autointent.context.data_handler import DataHandler
67
from autointent.modules import KNNScorer
78
from tests.conftest import get_test_embedder_config
@@ -45,3 +46,25 @@ def test_base_knn(dataset):
4546
new_scorer = KNNScorer.load(temp_dir)
4647
new_predictions = new_scorer.predict(test_data)
4748
assert np.allclose(predictions, new_predictions)
49+
50+
51+
def test_knn_in_pipeline(dataset):
52+
"""Test KNNScorer as part of an AutoML pipeline."""
53+
search_space = [
54+
{
55+
"node_type": "scoring",
56+
"search_space": [
57+
{
58+
"module_name": "knn",
59+
"k": [3],
60+
"weights": ["distance"],
61+
}
62+
],
63+
},
64+
{"node_type": "decision", "search_space": [{"module_name": "argmax"}]},
65+
]
66+
67+
pipeline = Pipeline.from_search_space(search_space)
68+
pipeline.fit(dataset)
69+
predictions = pipeline.predict(["test utterance"])
70+
assert len(predictions) == 1

‎tests/modules/scoring/test_linear.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import numpy as np
44

5+
from autointent import Pipeline
56
from autointent.context.data_handler import DataHandler
67
from autointent.modules import LinearScorer
78
from tests.conftest import get_test_embedder_config
@@ -45,3 +46,23 @@ def test_base_linear(dataset):
4546
new_scorer = LinearScorer.load(temp_dir)
4647
new_predictions = new_scorer.predict(test_data)
4748
np.testing.assert_almost_equal(predictions, new_predictions, decimal=5)
49+
50+
51+
def test_linear_in_pipeline(dataset):
52+
"""Test LinearScorer as part of an AutoML pipeline."""
53+
search_space = [
54+
{
55+
"node_type": "scoring",
56+
"search_space": [
57+
{
58+
"module_name": "linear",
59+
}
60+
],
61+
},
62+
{"node_type": "decision", "search_space": [{"module_name": "argmax"}]},
63+
]
64+
65+
pipeline = Pipeline.from_search_space(search_space)
66+
pipeline.fit(dataset)
67+
predictions = pipeline.predict(["test utterance"])
68+
assert len(predictions) == 1

0 commit comments

Comments
 (0)