remove oos utilities from everywhere

Author: voorhs

autointent/_dataset/_dataset.py

@@ -6,7 +6,7 @@
 from pathlib import Path
 from typing import Any, TypedDict
 
-from datasets import ClassLabel, Sequence, concatenate_datasets, get_dataset_config_names, load_dataset
+from datasets import ClassLabel, Sequence, get_dataset_config_names, load_dataset
 from datasets import Dataset as HFDataset
 
 from autointent.custom_types import LabelType, Split
@@ -39,7 +39,7 @@ class Dataset(dict[str, HFDataset]):
 
     def __init__(self, *args: Any, intents: list[Intent], **kwargs: Any) -> None:  # noqa: ANN401
         """
-        Initialize the dataset and configure OOS split if applicable.
+        Initialize the dataset.
 
         :param args: Positional arguments to initialize the dataset.
         :param intents: List of intents associated with the dataset.
@@ -54,10 +54,6 @@ def __init__(self, *args: Any, intents: list[Intent], **kwargs: Any) -> None:  #
         if self.multilabel:
             self._encode_labels()
 
-        oos_split = self._create_oos_split()
-        if oos_split is not None:
-            self[Split.OOS] = oos_split
-
     @property
     def multilabel(self) -> bool:
         """
@@ -144,7 +140,10 @@ def to_json(self, filepath: str | Path) -> None:
 
         :param filepath: The path to the file where the JSON data will be saved.
         """
-        with Path(filepath).open("w") as file:
+        path = Path(filepath)
+        if not path.parent.exists():
+            path.parent.mkdir(parents=True)
+        with path.open("w") as file:
             json.dump(self.to_dict(), file, indent=4, ensure_ascii=False)
 
     def push_to_hub(self, repo_id: str, private: bool = False) -> None:
@@ -204,15 +203,6 @@ def _encode_labels(self) -> "Dataset":
         self._encoded_labels = True
         return self
 
-    def _is_oos(self, sample: Sample) -> bool:
-        """
-        Check if a sample is out-of-scope.
-
-        :param sample: The sample to check.
-        :return: True if the sample is out-of-scope, False otherwise.
-        """
-        return sample["label"] is None
-
     def _to_multilabel(self, sample: Sample) -> Sample:
         """
         Convert a sample's label to multilabel format.
@@ -241,20 +231,6 @@ def _encode_label(self, sample: Sample) -> Sample:
         sample["label"] = one_hot_label
         return sample
 
-    def _create_oos_split(self) -> HFDataset | None:
-        """
-        Create an out-of-scope (OOS) split from the dataset.
-
-        :return: The OOS split if created, None otherwise.
-        """
-        oos_splits = [split.filter(self._is_oos) for split in self.values()]
-        oos_splits = [oos_split for oos_split in oos_splits if oos_split.num_rows]
-        if oos_splits:
-            for split_name, split in self.items():
-                self[split_name] = split.filter(lambda sample: not self._is_oos(sample))
-            return concatenate_datasets(oos_splits)
-        return None
-
     def _cast_label_feature(self) -> None:
         """Cast the label feature of the dataset to the appropriate type."""
         for split_name, split in self.items():

autointent/_dataset/_validation.py

@@ -66,18 +66,9 @@ def validate_dataset(self) -> "DatasetReader":
         ]
         splits = [split for split in splits if split]
 
-        n_classes = [self._get_n_classes(split) for split in splits]
-        if len(set(n_classes)) != 1:
-            message = (
-                f"Mismatch in number of classes across splits. Found class counts: {n_classes}. "
-                "Ensure all splits have the same number of classes."
-            )
-            raise ValueError(message)
-        if not n_classes[0]:
-            message = "Number of classes is zero or undefined. " "Ensure at least one class is present in the splits."
-            raise ValueError(message)
+        n_classes = self._validate_classes(splits)
 
-        self._validate_intents(n_classes[0])
+        self._validate_intents(n_classes)
 
         for split in splits:
             self._validate_split(split)
@@ -100,6 +91,20 @@ def _get_n_classes(self, split: list[Sample]) -> int:
                         classes.add(label)
         return len(classes)
 
+    def _validate_classes(self, splits: list[list[Sample]]) -> int:
+        """Validate that each split has all classes."""
+        n_classes = [self._get_n_classes(split) for split in splits]
+        if len(set(n_classes)) != 1:
+            message = (
+                f"Mismatch in number of classes across splits. Found class counts: {n_classes}. "
+                "Ensure all splits have the same number of classes."
+            )
+            raise ValueError(message)
+        if not n_classes[0]:
+            message = "Number of classes is zero or undefined. " "Ensure at least one class is present in the splits."
+            raise ValueError(message)
+        return n_classes[0]
+
     def _validate_intents(self, n_classes: int) -> "DatasetReader":
         """
         Validate the intents by checking their IDs for sequential order.
@@ -132,7 +137,8 @@ def _validate_split(self, split: list[Sample]) -> "DatasetReader":
         intent_ids = {intent.id for intent in self.intents}
         for sample in split:
             message = (
-                f"Sample with label {sample.label} references a non-existent intent ID. " f"Valid IDs are {intent_ids}."
+                f"Sample with label {sample.label} and utterance {sample.utterance[:10]}... "
+                f"references a non-existent intent ID. Valid IDs are {intent_ids}."
             )
             if isinstance(sample.label, int) and sample.label not in intent_ids:
                 raise ValueError(message)

autointent/context/data_handler/_data_handler.py

@@ -153,30 +153,6 @@ def test_labels(self, idx: int | None = None) -> list[LabelType]:
         split = f"{Split.TEST}_{idx}" if idx is not None else Split.TEST
         return cast(list[LabelType], self.dataset[split][self.dataset.label_feature])
 
-    def oos_utterances(self, idx: int | None = None) -> list[str]:
-        """
-        Retrieve out-of-scope (OOS) utterances from the dataset.
-
-        If the dataset contains out-of-scope samples, retrieves the utterances
-        from the specified OOS split index (if provided) or the primary OOS split.
-        Returns an empty list if no OOS samples are available in the dataset.
-
-        :param idx: Optional index for a specific OOS split.
-        :return: List of out-of-scope utterances, or an empty list if unavailable.
-        """
-        if self.has_oos_samples():
-            split = f"{Split.OOS}_{idx}" if idx is not None else Split.OOS
-            return cast(list[str], self.dataset[split][self.dataset.utterance_feature])
-        return []
-
-    def has_oos_samples(self) -> bool:
-        """
-        Check if there are out-of-scope samples.
-
-        :return: True if there are out-of-scope samples.
-        """
-        return any(split.startswith(Split.OOS) for split in self.dataset)
-
     def dump(self, filepath: str | Path) -> None:
         """
         Save the dataset splits and intents to a JSON file.
@@ -205,12 +181,7 @@ def _split(self, random_seed: int, split_train: bool) -> None:
         elif Split.VALIDATION in self.dataset:
             self._split_validation(random_seed)
 
-        if self.has_oos_samples():
-            self._split_oos(random_seed)
-
         for split in self.dataset:
-            if split.startswith(Split.OOS):
-                continue
             n_classes_split = self.dataset.get_n_classes(split)
             if n_classes_split != self.n_classes:
                 message = (
@@ -280,24 +251,3 @@ def _split_test(self, test_size: float, random_seed: int) -> None:
         )
         self.dataset.pop(f"{Split.TEST}_0")
         self.dataset.pop(f"{Split.TEST}_1")
-
-    def _split_oos(self, random_seed: int) -> None:
-        self.dataset[f"{Split.OOS}_0"], self.dataset[f"{Split.OOS}_1"] = (
-            self.dataset[Split.OOS]
-            .train_test_split(
-                test_size=0.2,
-                shuffle=True,
-                seed=random_seed,
-            )
-            .values()
-        )
-        self.dataset[f"{Split.OOS}_1"], self.dataset[f"{Split.OOS}_2"] = (
-            self.dataset[f"{Split.OOS}_1"]
-            .train_test_split(
-                test_size=0.5,
-                shuffle=True,
-                seed=random_seed,
-            )
-            .values()
-        )
-        self.dataset.pop(Split.OOS)

autointent/context/optimization_info/_data_models.py

@@ -42,10 +42,6 @@ class ScorerArtifact(Artifact):
     train_scores: NDArray[np.float64] | None = Field(None, description="Scorer outputs for train utterances")
     validation_scores: NDArray[np.float64] | None = Field(None, description="Scorer outputs for validation utterances")
     test_scores: NDArray[np.float64] | None = Field(None, description="Scorer outputs for test utterances")
-    oos_scores: dict[str, NDArray[np.float64]] | None = Field(
-        None,
-        description="Scorer outputs for out-of-scope utterances",
-    )
 
 
 class DecisionArtifact(Artifact):

autointent/context/optimization_info/_optimization_info.py

@@ -5,7 +5,7 @@
 """
 
 from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Any, Literal
+from typing import TYPE_CHECKING, Any
 
 import numpy as np
 from numpy.typing import NDArray
@@ -175,20 +175,6 @@ def get_best_test_scores(self) -> NDArray[np.float64] | None:
         best_scorer_artifact: ScorerArtifact = self._get_best_artifact(node_type=NodeType.scoring)  # type: ignore[assignment]
         return best_scorer_artifact.test_scores
 
-    def get_best_oos_scores(self, split: Literal["train", "validation", "test"]) -> NDArray[np.float64] | None:
-        """
-        Retrieve the out-of-scope scores from the best scorer node.
-
-        :param split: The data split for which to retrieve the OOS scores.
-            Must be one of "train", "validation", or "test".
-        :return: A numpy array containing OOS scores for the specified split,
-            or `None` if no OOS scores are available.
-        """
-        best_scorer_artifact: ScorerArtifact = self._get_best_artifact(node_type=NodeType.scoring)  # type: ignore[assignment]
-        if best_scorer_artifact.oos_scores is not None:
-            return best_scorer_artifact.oos_scores[split]
-        return best_scorer_artifact.oos_scores
-
     def dump_evaluation_results(self) -> dict[str, Any]:
         """
         Dump evaluation results for all nodes.

autointent/custom_types.py

@@ -58,12 +58,10 @@ class Split:
     :cvar str TRAIN: Training split.
     :cvar str VALIDATION: Validation split.
     :cvar str TEST: Testing split.
-    :cvar str OOS: Out-of-scope split.
     :cvar str INTENTS: Intents split.
     """
 
     TRAIN = "train"
     VALIDATION = "validation"
     TEST = "test"
-    OOS = "oos"
     INTENTS = "intents"

autointent/modules/abc/_decision.py

@@ -92,13 +92,4 @@ def get_decision_evaluation_data(
         message = f"No '{split}' scores found in the optimization info"
         raise ValueError(message)
 
-    oos_scores = context.optimization_info.get_best_oos_scores(split)
-    return_scores = scores
-    if oos_scores is not None:
-        oos_labels = (
-            [[0] * context.get_n_classes()] * len(oos_scores) if context.is_multilabel() else [-1] * len(oos_scores)  # type: ignore[list-item]
-        )
-        labels = np.concatenate([labels, np.array(oos_labels)])
-        return_scores = np.concatenate([scores, oos_scores])
-
-    return labels.tolist(), return_scores  # type: ignore[return-value]
+    return labels.tolist(), scores  # type: ignore[return-value]

autointent/modules/abc/_scoring.py

@@ -7,7 +7,6 @@
 
 from autointent import Context
 from autointent.context.optimization_info import ScorerArtifact
-from autointent.custom_types import Split
 from autointent.metrics import SCORING_METRICS_MULTICLASS, SCORING_METRICS_MULTILABEL
 from autointent.modules.abc import Module
 
@@ -44,14 +43,6 @@ def score(
 
         scores = self.predict(utterances)
 
-        self._oos_scores = None
-        if context.data_handler.has_oos_samples():
-            self._oos_scores = {
-                Split.TRAIN: self.predict(context.data_handler.oos_utterances(0)),
-                Split.VALIDATION: self.predict(context.data_handler.oos_utterances(1)),
-                Split.TEST: self.predict(context.data_handler.oos_utterances(2)),
-            }
-
         self._train_scores = self.predict(context.data_handler.train_utterances(1))
         self._validation_scores = self.predict(context.data_handler.validation_utterances(1))
         self._test_scores = self.predict(context.data_handler.test_utterances())
@@ -63,13 +54,12 @@ def get_assets(self) -> ScorerArtifact:
         """
         Retrieve assets generated during scoring.
 
-        :return: ScorerArtifact containing test scores and out-of-scope (OOS) scores.
+        :return: ScorerArtifact containing test, validation and test scores.
         """
         return ScorerArtifact(
             train_scores=self._train_scores,
             validation_scores=self._validation_scores,
             test_scores=self._test_scores,
-            oos_scores=self._oos_scores,
         )
 
     @abstractmethod

autointent/modules/regexp/_regexp.py

@@ -126,9 +126,6 @@ def score(
         # whether or not to omit utterances on next stages if they were detected with regexp module
         assets = {
             "test_matches": list(self.predict(context.data_handler.test_utterances())),
-            "oos_matches": None
-            if not context.data_handler.has_oos_samples()
-            else self.predict(context.data_handler.oos_utterances(2)),
         }
         if assets["test_matches"] is None:
             msg = "no matches found"

tests/context/datahandler/test_data_handler.py

@@ -151,7 +151,6 @@ def test_dataset_initialization(mapping):
         {"train": mock_split(), "validation": mock_split(), "validation_0": mock_split()},
         {"train": mock_split(), "validation": mock_split(), "validation_1": mock_split()},
         {"train": mock_split(), "validation": mock_split(), "validation_0": mock_split(), "validation_1": mock_split()},
-        {"train": mock_split(), "oos": mock_split()},
     ],
 )
 def test_dataset_validation(mapping):