AutoIntent/tests/pipeline/test_optimization.py at aa92bea6cb22ae59f2c116c9cb6c5f06ad9fde0c · deeppavlov/AutoIntent · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
import importlib.resources as ires
import os

import pytest

from autointent import Pipeline
from autointent.configs import DataConfig, LoggingConfig
from tests.conftest import get_search_space, setup_environment


@pytest.mark.parametrize(
    ("data_config", "refit_after"),
    [
        (DataConfig(scheme="ho", separation_ratio=None), False),
        (DataConfig(scheme="ho", separation_ratio=0.5), False),
        (DataConfig(scheme="cv", separation_ratio=None), False),
        (DataConfig(scheme="cv", separation_ratio=0.5), False),
        (DataConfig(scheme="ho", separation_ratio=None), True),
        (DataConfig(scheme="ho", separation_ratio=0.5), True),
        (DataConfig(scheme="cv", separation_ratio=None), True),
        (DataConfig(scheme="cv", separation_ratio=0.5), True),
    ],
)
def test_with_regex(dataset, data_config, refit_after):
    project_dir = setup_environment()
    search_space = get_search_space("regex")

    pipeline_optimizer = Pipeline.from_search_space(search_space)

    pipeline_optimizer.set_config(LoggingConfig(project_dir=project_dir, dump_modules=True, clear_ram=True))
    pipeline_optimizer.set_config(data_config)

    pipeline_optimizer.fit(dataset, refit_after=refit_after)


def test_no_node_separation(dataset_no_oos):
    project_dir = setup_environment()
    search_space = get_search_space("light")

    pipeline_optimizer = Pipeline.from_search_space(search_space)

    pipeline_optimizer.set_config(LoggingConfig(project_dir=project_dir, dump_modules=True, clear_ram=True))
    pipeline_optimizer.set_config(DataConfig(scheme="ho", separation_ratio=None))

    pipeline_optimizer.fit(dataset_no_oos, refit_after=False)


def test_full_config(dataset_no_oos):
    config_path = ires.files("tests.assets.configs").joinpath("full_training.yaml")
    pipeline_optimizer = Pipeline.from_optimization_config(config_path)
    pipeline_optimizer.fit(dataset_no_oos, refit_after=False)


@pytest.mark.parametrize(
    "sampler",
    ["tpe", "random"],
)
def test_bayes(dataset, sampler):
    project_dir = setup_environment()
    search_space = get_search_space("optuna")

    pipeline_optimizer = Pipeline.from_search_space(search_space)

    pipeline_optimizer.set_config(LoggingConfig(project_dir=project_dir, dump_modules=True, clear_ram=True))
    pipeline_optimizer.set_config(DataConfig(scheme="ho", separation_ratio=0.5))

    pipeline_optimizer.fit(dataset, refit_after=False, sampler=sampler)


@pytest.mark.parametrize(
    "task_type",
    ["multiclass", "multilabel", "description"],
)
def test_cv(dataset, task_type):
    project_dir = setup_environment()
    search_space = get_search_space(task_type)

    pipeline_optimizer = Pipeline.from_search_space(search_space)

    pipeline_optimizer.set_config(LoggingConfig(project_dir=project_dir, dump_modules=True, clear_ram=True))
    pipeline_optimizer.set_config(DataConfig(scheme="cv", separation_ratio=0.5))

    if task_type == "multilabel":
        dataset = dataset.to_multilabel()

    context = pipeline_optimizer.fit(dataset, refit_after=True)
    context.dump()

    assert os.listdir(pipeline_optimizer.logging_config.dump_dir)


@pytest.mark.parametrize(
    "task_type",
    ["multiclass", "multilabel", "description"],
)
def test_no_context_optimization(dataset, task_type):
    project_dir = setup_environment()
    search_space = get_search_space(task_type)

    pipeline_optimizer = Pipeline.from_search_space(search_space)

    pipeline_optimizer.set_config(LoggingConfig(project_dir=project_dir, dump_modules=False, clear_ram=False))
    pipeline_optimizer.set_config(DataConfig(scheme="ho", separation_ratio=0.5))

    if task_type == "multilabel":
        dataset = dataset.to_multilabel()

    context = pipeline_optimizer.fit(dataset)
    context.dump()


@pytest.mark.parametrize(
    "task_type",
    ["multiclass", "multilabel", "description"],
)
def test_dump_modules(dataset, task_type):
    project_dir = setup_environment()
    search_space = get_search_space(task_type)

    pipeline_optimizer = Pipeline.from_search_space(search_space)

    pipeline_optimizer.set_config(LoggingConfig(project_dir=project_dir, dump_modules=True, clear_ram=True))

    if task_type == "multilabel":
        dataset = dataset.to_multilabel()

    context = pipeline_optimizer.fit(dataset)
    context.dump()

    assert os.listdir(pipeline_optimizer.logging_config.dump_dir)