new presets and multiple bugs fix (#247)

* add presets

* lots of small fixes, updates and changes

* proper testing

* add transformers configs to presets

* set scoring_f1 target metric

* remove comments

* fix embedder hashing

* fix torch scorers `report_to`

* fix pipeline refitting

* fix presets

* fix `clear_cache` for torch models

* add extra exception handler into llm generator

* fix transformers heavy preset

* fix no-hpo

* remove rerank from medium

* upd classic heavy

* remove unnecessary event loop closing

* fix sklearn scorer cache clearing

* try to optimize bert scorer

* upd `report_to` in bert

* Update optimizer_config.schema.json

* fix typo

* try to fix typing

* fix callbacks test

* run formatter

* try to fix typing errors

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

Author: voorhs

autointent/_callbacks/tensorboard.py

@@ -16,7 +16,7 @@ def __init__(self) -> None:
         Raises an ImportError if neither are installed.
         """
         try:
-            from torch.utils.tensorboard import SummaryWriter  # type: ignore[attr-defined]
+            from torch.utils.tensorboard import SummaryWriter
 
             self.writer = SummaryWriter
         except ImportError:

autointent/_dump_tools/unit_dumpers.py

@@ -228,12 +228,12 @@ def load(path: Path, **kwargs: Any) -> PeftModel:  # noqa: ANN401, ARG004
         if (path / "ptuning").exists():
             # prompt learning model
             ptuning_path = path / "ptuning"
-            model = AutoModelForSequenceClassification.from_pretrained(ptuning_path / "base_model")  # type: ignore[no-untyped-call]
+            model = AutoModelForSequenceClassification.from_pretrained(ptuning_path / "base_model")
             return PeftModel.from_pretrained(model, ptuning_path / "peft")
         if (path / "lora").exists():
             # merged lora model
             lora_path = path / "lora"
-            return AutoModelForSequenceClassification.from_pretrained(lora_path)  # type: ignore[no-untyped-call,no-any-return]
+            return AutoModelForSequenceClassification.from_pretrained(lora_path)  # type: ignore[no-any-return]
         msg = f"Invalid PeftModel directory structure at {path}. Expected 'ptuning' or 'lora' subdirectory."
         raise ValueError(msg)
 
@@ -252,7 +252,7 @@ def dump(obj: PreTrainedModel, path: Path, exists_ok: bool) -> None:
 
     @staticmethod
     def load(path: Path, **kwargs: Any) -> PreTrainedModel:  # noqa: ANN401, ARG004
-        return AutoModelForSequenceClassification.from_pretrained(path)  # type: ignore[no-untyped-call,no-any-return]
+        return AutoModelForSequenceClassification.from_pretrained(path)  # type: ignore[no-any-return]
 
     @classmethod
     def check_isinstance(cls, obj: Any) -> bool:  # noqa: ANN401
@@ -269,7 +269,7 @@ def dump(obj: PreTrainedTokenizer | PreTrainedTokenizerFast, path: Path, exists_
 
     @staticmethod
     def load(path: Path, **kwargs: Any) -> PreTrainedTokenizer | PreTrainedTokenizerFast:  # noqa: ANN401, ARG004
-        return AutoTokenizer.from_pretrained(path)  # type: ignore[no-any-return]
+        return AutoTokenizer.from_pretrained(path)  # type: ignore[no-any-return,no-untyped-call]
 
     @classmethod
     def check_isinstance(cls, obj: Any) -> bool:  # noqa: ANN401

autointent/_pipeline/_pipeline.py

@@ -380,9 +380,11 @@ def _refit(self, context: Context) -> None:
 
         context.data_handler.prepare_for_refit()
 
+        scoring_module.clear_cache()
         scoring_module.fit(*scoring_module.get_train_data(context))
         scores = scoring_module.predict(context.data_handler.train_utterances(1))
 
+        decision_module.clear_cache()
         decision_module.fit(scores, context.data_handler.train_labels(1), context.data_handler.tags)
 
     def predict_with_metadata(self, utterances: list[str]) -> InferencePipelineOutput:

autointent/_presets/heavy.yaml autointent/_presets/classic-heavy.yamlautointent/_presets/heavy.yaml renamed to autointent/_presets/classic-heavy.yaml

@@ -1,7 +1,6 @@
-# TODO add sklearn RandomForestClassifier
 search_space:
   - node_type: scoring
-    target_metric: scoring_roc_auc
+    target_metric: scoring_f1
     search_space:
       - module_name: knn
         k:
@@ -13,24 +12,15 @@ search_space:
         k:
           low: 1
           high: 20
-      - module_name: description_bi
-        temperature:
-          low: 0.01
-          high: 10
-          log: true
-      - module_name: description_cross
-        temperature:
-          low: 0.01
-          high: 10
-          log: true
-      - module_name: rerank
-        k:
-          low: 10
-          high: 40
-        m:
-          low: 1
-          high: 10
-        weights: [uniform, distance, closest]
+      - module_name: catboost
+        depth: [3, 6, 10]
+        features_type: ["text", "embedding", "both"]
+      - module_name: sklearn
+        clf_name: [RandomForestClassifier]
+        n_estimators: [200, 300, 500]
+        max_depth: [50, 100, 150]
+        max_features: [sqrt, log2]
+        n_jobs: [8]
   - node_type: decision
     target_metric: decision_accuracy
     search_space:
@@ -44,5 +34,7 @@ search_space:
       - module_name: adaptive
 hpo_config:
   sampler: tpe
-  n_trials: 128 # dont know yet if its good
-  n_startup_trials: 32
+  n_trials: 55
+  n_startup_trials: 20
+embedder_config:
+  model_name: intfloat/multilingual-e5-large-instruct

autointent/_presets/light.yaml autointent/_presets/classic-light.yamlautointent/_presets/light.yaml renamed to autointent/_presets/classic-light.yaml

@@ -1,6 +1,6 @@
 search_space:
   - node_type: scoring
-    target_metric: scoring_roc_auc
+    target_metric: scoring_f1
     search_space:
       - module_name: knn
         k:
@@ -20,7 +20,12 @@ search_space:
           low: 0.1
           high: 0.9
       - module_name: argmax
+      - module_name: jinoos
+      - module_name: tunable
+      - module_name: adaptive
 hpo_config:
-  sampler: random
-  n_trials: 128 # dont know yet if its good
-  n_startup_trials: 32
+  sampler: tpe
+  n_trials: 20
+  n_startup_trials: 10
+embedder_config:
+  model_name: intfloat/multilingual-e5-large-instruct

autointent/_presets/classic-medium.yaml

@@ -0,0 +1,37 @@
+search_space:
+  - node_type: scoring
+    target_metric: scoring_f1
+    search_space:
+      - module_name: knn
+        k:
+          low: 1
+          high: 20
+        weights: [uniform, distance, closest]
+      - module_name: linear
+      - module_name: mlknn
+        k:
+          low: 1
+          high: 20
+      - module_name: catboost
+      - module_name: sklearn
+        clf_name: [RandomForestClassifier]
+        n_estimators: [150]
+        max_depth: [100]
+        n_jobs: [8]
+  - node_type: decision
+    target_metric: decision_accuracy
+    search_space:
+      - module_name: threshold
+        thresh:
+          low: 0.1
+          high: 0.9
+      - module_name: argmax
+      - module_name: jinoos
+      - module_name: tunable
+      - module_name: adaptive
+hpo_config:
+  sampler: tpe
+  n_trials: 20
+  n_startup_trials: 10
+embedder_config:
+  model_name: intfloat/multilingual-e5-large-instruct

autointent/_presets/nn-heavy.yaml

@@ -0,0 +1,44 @@
+search_space:
+  - node_type: scoring
+    target_metric: scoring_f1
+    search_space:
+      - module_name: cnn
+        dropout:
+          low: 0.1
+          high: 0.3
+        batch_size: [32, 64, 128]
+        learning_rate:
+          low: 5.0e-4
+          high: 1.0e-2
+          log: True
+        num_train_epochs: [60]
+        embed_dim: [64, 96, 128]
+        kernel_sizes: [[3, 4, 5]]
+        num_filters: [64, 96, 128]
+      - module_name: rnn
+        dropout:
+          low: 0.1
+          high: 0.3
+        batch_size: [32, 64, 128]
+        learning_rate:
+          low: 5.0e-4
+          high: 1.0e-2
+          log: True
+        num_train_epochs: [60]
+        embed_dim: [64, 96, 128]
+        hidden_dim: [128, 256, 512]
+  - node_type: decision
+    target_metric: decision_accuracy
+    search_space:
+      - module_name: threshold
+        thresh:
+          low: 0.1
+          high: 0.9
+      - module_name: argmax
+      - module_name: jinoos
+      - module_name: tunable
+      - module_name: adaptive
+hpo_config:
+  sampler: tpe
+  n_trials: 55
+  n_startup_trials: 20

autointent/_presets/nn-medium.yaml

@@ -0,0 +1,44 @@
+search_space:
+  - node_type: scoring
+    target_metric: scoring_f1
+    search_space:
+      - module_name: cnn
+        dropout:
+          low: 0.1
+          high: 0.3
+        batch_size: [32, 64, 128]
+        learning_rate:
+          low: 5.0e-4
+          high: 1.0e-2
+          log: True
+        num_train_epochs: [60]
+        embed_dim: [64]
+        kernel_sizes: [[3, 4, 5]]
+        num_filters: [64]
+      - module_name: rnn
+        dropout:
+          low: 0.1
+          high: 0.3
+        batch_size: [32, 64, 128]
+        learning_rate:
+          low: 5.0e-4
+          high: 1.0e-2
+          log: True
+        num_train_epochs: [60]
+        embed_dim: [64]
+        hidden_dim: [128]
+  - node_type: decision
+    target_metric: decision_accuracy
+    search_space:
+      - module_name: threshold
+        thresh:
+          low: 0.1
+          high: 0.9
+      - module_name: argmax
+      - module_name: jinoos
+      - module_name: tunable
+      - module_name: adaptive
+hpo_config:
+  sampler: tpe
+  n_trials: 55
+  n_startup_trials: 20

autointent/_presets/transformers-heavy.yaml

@@ -0,0 +1,28 @@
+search_space:
+  - node_type: scoring
+    target_metric: scoring_f1
+    search_space:
+      - module_name: bert
+        classification_model_config:
+          - model_name: microsoft/deberta-v3-large
+        num_train_epochs: [30]
+        batch_size: [32, 64]
+        learning_rate:
+          low: 1.0e-5
+          high: 1.0e-4
+          log: True
+  - node_type: decision
+    target_metric: decision_accuracy
+    search_space:
+      - module_name: threshold
+        thresh:
+          low: 0.1
+          high: 0.9
+      - module_name: argmax
+      - module_name: jinoos
+      - module_name: tunable
+      - module_name: adaptive
+hpo_config:
+  sampler: tpe
+  n_trials: 40
+  n_startup_trials: 20

autointent/_presets/transformers-light.yaml

@@ -0,0 +1,28 @@
+search_space:
+  - node_type: scoring
+    target_metric: scoring_f1
+    search_space:
+      - module_name: bert
+        classification_model_config:
+          - model_name: microsoft/deberta-v3-small
+        num_train_epochs: [30]
+        batch_size: [32, 64, 128]
+        learning_rate:
+          low: 1.0e-5
+          high: 1.0e-4
+          log: True
+  - node_type: decision
+    target_metric: decision_accuracy
+    search_space:
+      - module_name: threshold
+        thresh:
+          low: 0.1
+          high: 0.9
+      - module_name: argmax
+      - module_name: jinoos
+      - module_name: tunable
+      - module_name: adaptive
+hpo_config:
+  sampler: tpe
+  n_trials: 40
+  n_startup_trials: 20