Feat/augmentation utterances (#94)

* feat: added generation utterances

* feat: update generation

* feat: change prompt templates

* Refactor/move to our dataset class (#100)

* refactor basic utterance generator

* make `load_dataset` utility public

* polish `load_dataset` utility

* move basic utterance generator to `Dataset`

* refactor cli for basic utterance generator

* refactor evolutions module

* some bug fix in basic utterance generation

* some bug fix in evolutionary augmentations

* refactor `Generator` and fix codestyle

* fix typing

* fix import issues

* try to fix

---------

Co-authored-by: Алексеев Илья <[email protected]>
Co-authored-by: voorhs <[email protected]>

Author: Darinochka

autointent/__init__.py

@@ -6,7 +6,18 @@
 from ._vector_index import VectorIndex
 from ._dataset import Dataset
 from ._hash import Hasher
-from .context import Context
+from .context import Context, load_dataset
 from ._pipeline import Pipeline
 
-__all__ = ["Context", "Dataset", "Embedder", "Hasher", "Pipeline", "Ranker", "VectorIndex", "setup_logging"]
+
+__all__ = [
+    "Context",
+    "Dataset",
+    "Embedder",
+    "Hasher",
+    "Pipeline",
+    "Ranker",
+    "VectorIndex",
+    "load_dataset",
+    "setup_logging",
+]

autointent/context/__init__.py

@@ -1,5 +1,6 @@
 """Core utilities for auto ML features."""
 
 from ._context import Context
+from ._utils import load_dataset
 
-__all__ = ["Context"]
+__all__ = ["Context", "load_dataset"]

autointent/context/_context.py

@@ -17,7 +17,7 @@
     VectorIndexConfig,
 )
 
-from ._utils import NumpyEncoder, load_data
+from ._utils import NumpyEncoder, load_dataset
 from .data_handler import DataHandler
 from .optimization_info import OptimizationInfo
 
@@ -81,7 +81,7 @@ def configure_data(self, config: DataConfig) -> None:
         :param config: Configuration for the data handling process.
         """
         self.data_handler = DataHandler(
-            dataset=load_data(config.train_path),
+            dataset=load_dataset(config.train_path),
             random_seed=self.seed,
         )
 

autointent/context/_utils.py

@@ -37,9 +37,9 @@ def default(self, obj: Any) -> str | int | float | list[Any] | Any:  # noqa: ANN
         return super().default(obj)
 
 
-def load_data(filepath: str | Path) -> Dataset:
+def load_dataset(path: str | Path) -> Dataset:
     """
-    Load data from a specified path or use default sample data.
+    Load data from a specified path or use default sample data or load from hugging face hub.
 
     This function loads a dataset from a JSON file or retrieves sample data
     included with the `autointent` package for default multiclass or multilabel
@@ -50,10 +50,10 @@ def load_data(filepath: str | Path) -> Dataset:
                       - "default-multilabel": Loads sample multilabel dataset.
     :return: A `Dataset` object containing the loaded data.
     """
-    if filepath == "default-multiclass":
+    if path == "default-multiclass":
         return Dataset.from_hub("AutoIntent/clinc150_subset")
-    if filepath == "default-multilabel":
+    if path == "default-multilabel":
         return Dataset.from_hub("AutoIntent/clinc150_subset").to_multilabel()
-    if not Path(filepath).exists():
-        return Dataset.from_hub(str(filepath))
-    return Dataset.from_json(filepath)
+    if not Path(path).exists():
+        return Dataset.from_hub(str(path))
+    return Dataset.from_json(path)

autointent/generation/__init__.py

@@ -1 +0,0 @@
-"""Experimental subpackage that someday will evolve into data augmentation tools."""

autointent/generation/intents/__init__.py

@@ -7,7 +7,7 @@
 from openai import AsyncOpenAI
 
 from autointent import Dataset
-from autointent.generation.prompt_scheme import PromptDescription
+from autointent.generation.intents.prompt_scheme import PromptDescription
 from autointent.schemas import Intent, Sample
 
 

…ent/generation/description_generation.py …ration/intents/description_generation.pyautointent/generation/description_generation.py renamed to autointent/generation/intents/description_generation.py autointent/generation/description_generation.py renamed to autointent/generation/intents/description_generation.py

@@ -2,7 +2,7 @@
 
 from pydantic import BaseModel, field_validator
 
-from autointent.generation.prompts import PROMPT_DESCRIPTION
+from autointent.generation.utterances.prompts import PROMPT_DESCRIPTION
 
 
 class PromptDescription(BaseModel):