Test/regex module (#156)

* forbid extra field in pydantic configs

* bug fix for regex module

* add integration test for regex module

* fix codestyle

* regex bug fix in validation logic

* update test for regex

* fix typing

* Update optimizer_config.schema.json

* update tests

* change cv logic a little bit

* fix `train_classifier` issue

* fix docs building

* fix docs building

* try another

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

Author: voorhs

autointent/_pipeline/_pipeline.py

@@ -60,7 +60,7 @@ def __init__(
         self.sampler = sampler
 
         if isinstance(nodes[0], NodeOptimizer):
-            self.logging_config = LoggingConfig(dump_dir=None)
+            self.logging_config = LoggingConfig()
             self.embedder_config = EmbedderConfig()
             self.cross_encoder_config = CrossEncoderConfig()
             self.data_config = DataConfig()

autointent/_ranker.py

@@ -31,14 +31,14 @@ class CrossEncoderMetadata(TypedDict):
 
     Attributes:
         model_name: Name of the model
-        train_classifier: Whether to train a classifier
+        train_head: Whether to train a classifier
         device: Device to use for inference
         max_length: Maximum sequence length
         batch_size: Batch size for inference
     """
 
     model_name: str
-    train_classifier: bool
+    train_head: bool
     device: str | None
     max_length: int | None
     batch_size: int
@@ -119,11 +119,11 @@ def __init__(
             device=self.cross_encoder_config.device,
             max_length=self.cross_encoder_config.max_length,  # type: ignore[arg-type]
         )
-        self.train_classifier = False
+        self.train_head = False
         self._clf = classifier_head
 
         if classifier_head is not None or self.cross_encoder_config.train_head:
-            self.train_classifier = True
+            self.train_head = True
             self._activations_list: list[npt.NDArray[Any]] = []
             self._hook_handler = self.cross_encoder.model.classifier.register_forward_hook(self._classifier_hook)
 
@@ -147,7 +147,7 @@ def _get_features_or_predictions(self, pairs: list[tuple[str, str]]) -> npt.NDAr
         Returns:
             Array of extracted features or predictions
         """
-        if not self.train_classifier:
+        if not self.train_head:
             return np.array(
                 self.cross_encoder.predict(
                     pairs,
@@ -189,7 +189,7 @@ def fit(self, utterances: list[str], labels: ListOfLabels) -> None:
             utterances: List of utterances (texts)
             labels: Intent class labels corresponding to the utterances
         """
-        if not self.train_classifier:
+        if not self.train_head:
             return
 
         pairs, labels_ = construct_samples(utterances, labels, balancing_factor=1)
@@ -207,7 +207,7 @@ def predict(self, pairs: list[tuple[str, str]]) -> npt.NDArray[Any]:
         Raises:
             ValueError: If classifier is not trained yet
         """
-        if self.train_classifier and self._clf is None:
+        if self.train_head and self._clf is None:
             msg = "Classifier is not trained yet"
             raise ValueError(msg)
 
@@ -254,7 +254,7 @@ def save(self, path: str) -> None:
 
         metadata = CrossEncoderMetadata(
             model_name=self.cross_encoder_config.model_name,
-            train_classifier=self.train_classifier,
+            train_head=self.train_head,
             device=self.cross_encoder_config.device,
             max_length=self.cross_encoder_config.max_length,
             batch_size=self.cross_encoder_config.batch_size,

autointent/configs/_optimization.py

@@ -2,7 +2,7 @@
 
 from pathlib import Path
 
-from pydantic import BaseModel, Field, PositiveInt
+from pydantic import BaseModel, ConfigDict, Field, PositiveInt
 
 from autointent._callbacks import REPORTERS_NAMES
 from autointent.custom_types import FloatFromZeroToOne, ValidationScheme
@@ -13,6 +13,7 @@
 class DataConfig(BaseModel):
     """Configuration for the data used in the optimization process."""
 
+    model_config = ConfigDict(extra="forbid")
     scheme: ValidationScheme = Field("ho", description="Validation scheme to use.")
     """Hold-out or cross-validation."""
     n_folds: PositiveInt = Field(3, description="Number of folds in cross-validation.")
@@ -33,6 +34,8 @@ class DataConfig(BaseModel):
 class LoggingConfig(BaseModel):
     """Configuration for the logging."""
 
+    model_config = ConfigDict(extra="forbid")
+
     _dirpath: Path | None = None
     _dump_dir: Path | None = None
 

autointent/configs/_transformers.py

@@ -1,15 +1,12 @@
 from enum import Enum
 from typing import Any
 
-from pydantic import (
-    BaseModel,
-    Field,
-    PositiveInt,
-)
+from pydantic import BaseModel, ConfigDict, Field, PositiveInt
 from typing_extensions import Self, assert_never
 
 
 class ModelConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
     batch_size: PositiveInt = Field(32, description="Batch size for model inference.")
     max_length: PositiveInt | None = Field(None, description="Maximum length of input sequences.")
 

autointent/modules/regex/_simple.py

@@ -1,12 +1,16 @@
 """Module for regular expressions based intent detection."""
 
 import re
+from collections.abc import Iterable
 from typing import Any, TypedDict
 
+import numpy as np
+import numpy.typing as npt
+
 from autointent import Context
 from autointent.context.data_handler._data_handler import RegexPatterns
 from autointent.context.optimization_info import Artifact
-from autointent.custom_types import LabelType
+from autointent.custom_types import LabelType, ListOfGenericLabels, ListOfLabels
 from autointent.metrics import REGEX_METRICS
 from autointent.modules.base import BaseRegex
 from autointent.schemas import Intent
@@ -36,7 +40,10 @@ class Regex(BaseRegex):
         name: Name of the module, defaults to "regex"
     """
 
-    name = "regex"
+    name = "simple"
+    supports_multiclass = True
+    supports_multilabel = True
+    supports_oos = False
 
     @classmethod
     def from_context(cls, context: Context) -> "Regex":
@@ -158,7 +165,7 @@ def score_ho(self, context: Context, metrics: list[str]) -> dict[str, float]:
         return self.score_metrics_ho((val_labels, pred_labels), chosen_metrics)
 
     def score_cv(self, context: Context, metrics: list[str]) -> dict[str, float]:
-        """Score the model using cross-validation.
+        """Score the model in cross-validation mode.
 
         Args:
             context: Context containing validation data
@@ -169,10 +176,42 @@ def score_cv(self, context: Context, metrics: list[str]) -> dict[str, float]:
         """
         chosen_metrics = {name: fn for name, fn in REGEX_METRICS.items() if name in metrics}
 
-        metrics_calculated, _ = self.score_metrics_cv(chosen_metrics, context.data_handler.validation_iterator())
+        metrics_calculated, _ = self.score_metrics_cv(
+            chosen_metrics, context.data_handler.validation_iterator(), intents=context.data_handler.dataset.intents
+        )
 
         return metrics_calculated
 
+    def score_metrics_cv(
+        self,
+        metrics_dict: dict[str, Any],
+        cv_iterator: Iterable[tuple[list[str], ListOfLabels, list[str], ListOfLabels]],
+        intents: list[Intent],
+    ) -> tuple[dict[str, float], list[ListOfGenericLabels] | list[npt.NDArray[Any]]]:
+        """Score metrics using cross-validation.
+
+        Args:
+            metrics_dict: Dictionary of metrics to compute
+            cv_iterator: Cross-validation iterator
+            intents: intents from the dataset
+
+        Returns:
+            Tuple of metrics dictionary and predictions
+        """
+        metrics_values: dict[str, list[float]] = {name: [] for name in metrics_dict}
+        all_val_preds = []
+
+        self.fit(intents)
+
+        for _, _, val_utterances, val_labels in cv_iterator:
+            val_preds = self.predict(val_utterances)
+            for name, fn in metrics_dict.items():
+                metrics_values[name].append(fn(val_labels, val_preds))
+            all_val_preds.append(val_preds)
+
+        metrics = {name: float(np.mean(values_list)) for name, values_list in metrics_values.items()}
+        return metrics, all_val_preds  # type: ignore[return-value]
+
     def clear_cache(self) -> None:
         """Clear cached regex patterns."""
         del self.regex_patterns

autointent/modules/scoring/_dnnc/dnnc.py

@@ -52,13 +52,6 @@ class DNNCScorer(BaseScorer):
 
         test_utterances = ["Hello!", "What's up?"]
         scores = scorer.predict(test_utterances)
-        print(scores)  # Outputs similarity scores for the utterances
-
-
-    .. testoutput::
-
-        [[0.00013581 0.        ]
-         [0.00030066 0.        ]]
 
     """
 

autointent/modules/scoring/_knn/knn.py

@@ -42,12 +42,6 @@ class KNNScorer(BaseScorer):
         scorer.fit(utterances, labels)
         test_utterances = ["hi", "what's up?"]
         probabilities = scorer.predict(test_utterances)
-        print(probabilities)  # Outputs predicted class probabilities for the utterances
-
-    .. testoutput::
-
-        [[0.67297815 0.32702185]
-         [0.44031667 0.55968333]]
 
     """
 

autointent/nodes/info/_regex.py

@@ -6,8 +6,8 @@
 from autointent.custom_types import NodeType
 from autointent.metrics import REGEX_METRICS
 from autointent.metrics.regex import RegexMetricFn
+from autointent.modules import REGEX_MODULES
 from autointent.modules.base import BaseRegex
-from autointent.modules.regex import Regex
 
 from ._base import NodeInfo
 
@@ -17,6 +17,6 @@ class RegexNodeInfo(NodeInfo):
 
     metrics_available: ClassVar[Mapping[str, RegexMetricFn]] = REGEX_METRICS
 
-    modules_available: ClassVar[Mapping[str, type[BaseRegex]]] = {NodeType.regex: Regex}
+    modules_available: ClassVar[Mapping[str, type[BaseRegex]]] = REGEX_MODULES
 
     node_type = NodeType.regex

docs/optimizer_config.schema.json

@@ -1,6 +1,7 @@
 {
     "$defs": {
         "CrossEncoderConfig": {
+            "additionalProperties": false,
             "properties": {
                 "batch_size": {
                     "default": 32,
@@ -53,6 +54,7 @@
             "type": "object"
         },
         "DataConfig": {
+            "additionalProperties": false,
             "description": "Configuration for the data used in the optimization process.",
             "properties": {
                 "scheme": {
@@ -100,6 +102,7 @@
             "type": "object"
         },
         "EmbedderConfig": {
+            "additionalProperties": false,
             "properties": {
                 "batch_size": {
                     "default": 32,
@@ -230,6 +233,7 @@
             "type": "object"
         },
         "LoggingConfig": {
+            "additionalProperties": false,
             "description": "Configuration for the logging.",
             "properties": {
                 "project_dir": {

tests/assets/configs/regex.yaml

@@ -0,0 +1,14 @@
+- node_type: regex
+  target_metric: regex_partial_accuracy
+  search_space:
+    - module_name: simple
+- node_type: scoring
+  target_metric: scoring_roc_auc
+  search_space:
+    - module_name: linear
+      embedder_config:
+        - model_name: sentence-transformers/all-MiniLM-L6-v2
+- node_type: decision
+  target_metric: decision_accuracy
+  search_space:
+    - module_name: argmax