Feat/basic presets (#146)

* move `EmbedderConfig` and `CrossEncoderConfig` to `autointent.configs`

* remove `_datafiles`

* implement search space filtering

* add `warning` mode

* change default model for cross-encoder

* refactor constructor of `SklearnScorer`

* add the heaviest search space preset

* add search space validation for `DescriptionScorer`

* add default value for `ThresholdDecision`

* implement default configs for embedder and cross encoder

* add transformers configs to pipeline

* implement two basic presets

* fix codestyle

* fix typing

* Update optimizer_config.schema.json

* bug fix and update test

* search space validation bug found

* update unit tests

* update test

* improve sklearn test

* refactor sklearn scorer

* remove `VectorIndexConfig` entirely from our lib

* try to fix validation errors

* remove multiclass/multilabel separation on modules dicts

* upd test

* fix codestyle

* Update optimizer_config.schema.json

* something's wrong with sklearn scorer again

* remove unnecessary default value from sklearn scorer constructor

* add default value for `weights` in knn and rerank scorer

* try without search space validation

* Update optimizer_config.schema.json

* foolish bug fix

* Update optimizer_config.schema.json

* finish implementing presets

* Update optimizer_config.schema.json

* update docs

* update docs and readme

* upd ci

* respond to samoed

* update tests

* fix codestyle

* Update optimizer_config.schema.json

* remove sklearn scorer from config for now

* bug fix

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

Author: voorhs

.github/workflows/test-presets.yaml

@@ -0,0 +1,38 @@
+name: test presets
+
+on:
+  push:
+    branches:
+      - dev
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest ]
+        python-version: [ "3.10", "3.11", "3.12" ]
+        include:
+          - os: windows-latest
+            python-version: "3.10"
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Setup Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: "pip"
+
+    - name: Install dependencies
+      run: |
+          pip install .
+          pip install pytest pytest-asyncio
+
+    - name: Run tests
+      run: |
+        pytest tests/pipeline/test_presets.py

README.md

@@ -30,7 +30,7 @@ Example of building an intent classifier in a couple of lines of code:
 from autointent import Pipeline, Dataset
 
 dataset = Dataset.from_json(path_to_json)
-pipeline = Pipeline.default_optimizer(multilabel=False)
+pipeline = Pipeline.from_preset("light")
 pipeline.fit(dataset)
-pipeline.predict(["show me my latest recent transactions"])
+pipeline.predict(["show me my latest transactions"])
 ```

autointent/__init__.py

@@ -7,6 +7,7 @@
 from ._dataset import Dataset
 from ._hash import Hasher
 from .context import Context, load_dataset
+from ._optimization_config import OptimizationConfig
 from ._pipeline import Pipeline
 
 
@@ -15,6 +16,7 @@
     "Dataset",
     "Embedder",
     "Hasher",
+    "OptimizationConfig",
     "Pipeline",
     "Ranker",
     "VectorIndex",

autointent/_datafiles/default-multiclass-config.yaml

@@ -1,6 +1,7 @@
 """File with Dataset definition."""
 
 import json
+import logging
 from collections import defaultdict
 from functools import cached_property
 from pathlib import Path
@@ -12,6 +13,8 @@
 from autointent.custom_types import LabelWithOOS, Split
 from autointent.schemas import Intent, Tag
 
+logger = logging.getLogger(__name__)
+
 
 class Sample(TypedDict):
     """
@@ -36,6 +39,7 @@ class Dataset(dict[str, HFDataset]):
 
     label_feature = "label"
     utterance_feature = "utterance"
+    has_descriptions: bool
 
     def __init__(self, *args: Any, intents: list[Intent], **kwargs: Any) -> None:  # noqa: ANN401
         """
@@ -49,6 +53,8 @@ def __init__(self, *args: Any, intents: list[Intent], **kwargs: Any) -> None:  #
 
         self.intents = intents
 
+        self.has_descriptions = self.validate_descriptions()
+
     @property
     def multilabel(self) -> bool:
         """
@@ -197,3 +203,18 @@ def _to_multilabel(self, sample: Sample) -> Sample:
             ohe_vector[sample["label"]] = 1
             sample["label"] = ohe_vector
         return sample
+
+    def validate_descriptions(self) -> bool:
+        """
+        Check whether the dataset contains text descriptions for each intent.
+
+        :return: True if all intents have description field
+        """
+        has_any = any(intent.description is not None for intent in self.intents)
+        has_all = all(intent.description is not None for intent in self.intents)
+
+        if has_any and not has_all:
+            msg = "Some intents have text descriptions, but some of them not."
+            logger.warning(msg)
+
+        return has_all

autointent/_datafiles/default-multilabel-config.yaml

@@ -12,7 +12,8 @@
 from sklearn.base import BaseEstimator
 
 from autointent import Embedder, Ranker, VectorIndex
-from autointent.schemas import CrossEncoderConfig, EmbedderConfig, TagsList
+from autointent.configs import CrossEncoderConfig, EmbedderConfig
+from autointent.schemas import TagsList
 
 ModuleSimpleAttributes = None | str | int | float | bool | list  # type: ignore[type-arg]
 

autointent/_datafiles/inference-config-example.yaml

@@ -17,7 +17,7 @@
 from sentence_transformers import SentenceTransformer
 
 from ._hash import Hasher
-from .schemas import EmbedderConfig, TaskTypeEnum
+from .configs import EmbedderConfig, TaskTypeEnum
 
 
 def get_embeddings_path(filename: str) -> Path:

autointent/_dataset/_dataset.py

@@ -0,0 +1,17 @@
+from pydantic import BaseModel, PositiveInt
+
+from .configs import CrossEncoderConfig, DataConfig, EmbedderConfig, LoggingConfig
+from .custom_types import SamplerType
+from .nodes.schemes import OptimizationSearchSpaceConfig
+
+
+class OptimizationConfig(BaseModel):
+    """Configuration for the optimization process."""
+
+    data_config: DataConfig = DataConfig()
+    search_space: OptimizationSearchSpaceConfig
+    logging_config: LoggingConfig = LoggingConfig()
+    embedder_config: EmbedderConfig = EmbedderConfig()
+    cross_encoder_config: CrossEncoderConfig = CrossEncoderConfig()
+    sampler: SamplerType = "brute"
+    seed: PositiveInt = 42