resolve conflicts

Author: voorhs

autointent/context/embedder.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import shutil
 from pathlib import Path
 from typing import TypedDict
 
@@ -37,12 +38,19 @@ def __init__(
 
         self.logger = logging.getLogger(__name__)
 
-    def delete(self) -> None:
+    def clear_ram(self) -> None:
         self.logger.debug("deleting embedder %s", self.model_name)
         self.embedding_model.cpu()
         del self.embedding_model
 
+    def delete(self) -> None:
+        self.clear_ram()
+        shutil.rmtree(
+            self.dump_dir, ignore_errors=True
+        )  # TODO: `ignore_errors=True` is workaround for PermissionError: [WinError 5] Access is denied
+
     def dump(self, path: Path) -> None:
+        self.dump_dir = path
         metadata = EmbedderDumpMetadata(
             batch_size=self.batch_size,
             max_length=self.max_length,
@@ -53,6 +61,7 @@ def dump(self, path: Path) -> None:
             json.dump(metadata, file, indent=4)
 
     def load(self, path: Path | str) -> None:
+        self.dump_dir = Path(path)
         path = Path(path)
         with (path / self.metadata_dict_name).open() as file:
             metadata: EmbedderDumpMetadata = json.load(file)
@@ -71,4 +80,9 @@ def embed(self, utterances: list[str]) -> npt.NDArray[np.float32]:
         )
         if self.max_length is not None:
             self.embedding_model.max_seq_length = self.max_length
-        return self.embedding_model.encode(utterances, convert_to_numpy=True, batch_size=self.batch_size)  # type: ignore[return-value]
+        return self.embedding_model.encode(
+            utterances,
+            convert_to_numpy=True,
+            batch_size=self.batch_size,
+            normalize_embeddings=True,
+        )  # type: ignore[return-value]

autointent/context/vector_index_client/cache.py

@@ -3,14 +3,6 @@
 
 
 def get_db_dir(db_dir: str | Path | None = None) -> Path:
-    """
-    Get the directory path for chroma db file.
-    Use default cache dir if not provided.
-    Save path into user config in order to remove it from cache later.
-    """
-
-    root = Path(db_dir) if db_dir is not None else Path.cwd()
-    db_dir = root / "vector_db" / str(uuid4()) if db_dir is None else Path(db_dir)
+    db_dir = Path.cwd() / ("vector_db_" + str(uuid4())) if db_dir is None else Path(db_dir)
     db_dir.mkdir(parents=True, exist_ok=True)
-
     return db_dir

autointent/context/vector_index_client/vector_index.py

@@ -47,13 +47,18 @@ def is_empty(self) -> bool:
 
     def delete(self) -> None:
         self.logger.debug("deleting vector index %s", self.model_name)
-        if hasattr(self, "index"):
-            self.index.reset()
+        self.embedder.delete()
+        self.clear_ram()
+        (self.dump_dir / "index.faiss").unlink()
+        (self.dump_dir / "texts.json").unlink()
+        (self.dump_dir / "labels.json").unlink()
+
+    def clear_ram(self) -> None:
+        self.logger.debug("clearing vector index %s from ram", self.model_name)
+        self.index.reset()
         self.labels = []
         self.texts = []
 
-        self.embedder.delete()
-
     def _search_by_text(self, texts: list[str], k: int) -> list[list[dict[str, Any]]]:
         query_embedding: npt.NDArray[np.float64] = self.embedder.embed(texts)  # type: ignore[assignment]
         return self._search_by_embedding(query_embedding, k)
@@ -122,11 +127,8 @@ def dump(self, dir_path: Path) -> None:
         with (self.dump_dir / "labels.json").open("w") as file:
             json.dump(self.labels, file, indent=4, ensure_ascii=False)
 
-    def load(self, dir_path: Path | None = None) -> None:
-        self.delete()
-
-        if dir_path is None:
-            dir_path = self.dump_dir
+    def load(self, dir_path: Path) -> None:
+        self.dump_dir = Path(dir_path)
         self.index = faiss.read_index(str(dir_path / "index.faiss"))
         self.embedder = Embedder(model_name=dir_path / "embedding_model", device=self.device)
         with (dir_path / "texts.json").open() as file:

autointent/context/vector_index_client/vector_index_client.py

@@ -1,6 +1,5 @@
 import json
 import logging
-import shutil
 from pathlib import Path
 
 from autointent.custom_types import LabelType
@@ -87,10 +86,10 @@ def _get_dump_dirpath(self, model_name: str) -> Path:
         return self.db_dir / dir_name
 
     def delete_index(self, model_name: str) -> None:
-        dir_name = self._remove_index_dirname(model_name)
-        if dir_name is not None:
-            self._logger.debug("Deleting index for model: %s", model_name)
-            shutil.rmtree(self.db_dir / dir_name)
+        if not self.exists(model_name):
+            return
+        index = self.get_index(model_name)
+        index.delete()
 
     def get_index(self, model_name: str) -> VectorIndex:
         dirpath = self._get_index_dirpath(model_name)
@@ -107,7 +106,14 @@ def exists(self, model_name: str) -> bool:
         return self._get_index_dirpath(model_name) is not None
 
     def delete_db(self) -> None:
-        shutil.rmtree(self.db_dir)
+        path = self.db_dir / "indexes_dirnames.json"
+        if not path.exists():
+            return
+        with path.open() as file:
+            indexes_dirnames: DIRNAMES_TYPE = json.load(file)
+        for embedder_name in indexes_dirnames:
+            self.delete_index(embedder_name)
+        path.unlink()
 
 
 class NonExistingIndexError(Exception):

autointent/metrics/scoring.py

@@ -24,7 +24,7 @@ def __call__(self, labels: LABELS_VALUE_TYPE, scores: SCORES_VALUE_TYPE) -> floa
         ...
 
 
-def scoring_log_likelihood(labels: LABELS_VALUE_TYPE, scores: SCORES_VALUE_TYPE) -> float:
+def scoring_log_likelihood(labels: LABELS_VALUE_TYPE, scores: SCORES_VALUE_TYPE, eps: float = 1e-10) -> float:
     """
     supports multiclass and multilabel
 
@@ -45,9 +45,10 @@ def scoring_log_likelihood(labels: LABELS_VALUE_TYPE, scores: SCORES_VALUE_TYPE)
     where `s[i,c]` is a predicted score of `i`th utterance having ground truth label `c`
     """
     labels_array, scores_array = transform(labels, scores)
+    scores_array[scores_array == 0] = eps
 
     if np.any((scores_array <= 0) | (scores_array > 1)):
-        msg = "One or more scores are not from [0,1]. It is incompatible with `scoring_log_likelihood` metric"
+        msg = "One or more scores are not from (0,1]. It is incompatible with `scoring_log_likelihood` metric"
         logger.error(msg)
         raise ValueError(msg)
 

autointent/modules/retrieval/vectordb.py

@@ -32,11 +32,9 @@ def __init__(
         batch_size: int = 32,
         max_length: int | None = None,
     ) -> None:
-        if db_dir is None:
-            db_dir = str(get_db_dir())
         self.embedder_name = embedder_name
         self.device = device
-        self.db_dir = db_dir
+        self._db_dir = db_dir
         self.batch_size = batch_size
         self.max_length = max_length
 
@@ -58,6 +56,12 @@ def from_context(
             max_length=context.get_max_length(),
         )
 
+    @property
+    def db_dir(self) -> str:
+        if self._db_dir is None:
+            self._db_dir = str(get_db_dir())
+        return self._db_dir
+
     def fit(self, utterances: list[str], labels: list[LabelType]) -> None:
         vector_index_client = VectorIndexClient(
             self.device, self.db_dir, embedder_batch_size=self.batch_size, embedder_max_length=self.max_length
@@ -76,7 +80,7 @@ def get_assets(self) -> RetrieverArtifact:
         return RetrieverArtifact(embedder_name=self.embedder_name)
 
     def clear_cache(self) -> None:
-        self.vector_index.delete()
+        self.vector_index.clear_ram()
 
     def dump(self, path: str) -> None:
         self.metadata = VectorDBMetadata(

autointent/modules/scoring/description/description.py

@@ -11,7 +11,6 @@
 from autointent.context import Context
 from autointent.context.embedder import Embedder
 from autointent.context.vector_index_client import VectorIndex, VectorIndexClient
-from autointent.context.vector_index_client.cache import get_db_dir
 from autointent.custom_types import LabelType
 from autointent.modules.scoring.base import ScoringModule
 
@@ -30,22 +29,19 @@ class DescriptionScorer(ScoringModule):
     precomputed_embeddings: bool = False
     embedding_model_subdir: str = "embedding_model"
     _vector_index: VectorIndex
+    db_dir: str
     name = "description"
 
     def __init__(
         self,
         embedder_name: str,
-        db_dir: Path | None = None,
         temperature: float = 1.0,
         device: str = "cpu",
         batch_size: int = 32,
         max_length: int | None = None,
     ) -> None:
-        if db_dir is None:
-            db_dir = get_db_dir()
         self.temperature = temperature
         self.device = device
-        self.db_dir = db_dir
         self.embedder_name = embedder_name
         self.batch_size = batch_size
         self.max_length = max_length
@@ -66,10 +62,10 @@ def from_context(
         instance = cls(
             temperature=temperature,
             device=context.get_device(),
-            db_dir=context.get_db_dir(),
             embedder_name=embedder_name,
         )
         instance.precomputed_embeddings = precomputed_embeddings
+        instance.db_dir = str(context.get_db_dir())
         return instance
 
     def get_embedder_name(self) -> str:
@@ -127,7 +123,7 @@ def predict(self, utterances: list[str]) -> NDArray[np.float64]:
         return probabilites  # type: ignore[no-any-return]
 
     def clear_cache(self) -> None:
-        self.embedder.delete()
+        self.embedder.clear_ram()
 
     def dump(self, path: str) -> None:
         self.metadata = DescriptionScorerDumpMetadata(

autointent/modules/scoring/dnnc/dnnc.py

@@ -52,18 +52,21 @@ def __init__(
         batch_size: int = 32,
         max_length: int | None = None,
     ) -> None:
-        if db_dir is None:
-            db_dir = str(get_db_dir())
-
         self.cross_encoder_name = cross_encoder_name
         self.embedder_name = embedder_name
         self.k = k
         self.train_head = train_head
         self.device = device
-        self.db_dir = db_dir
+        self._db_dir = db_dir
         self.batch_size = batch_size
         self.max_length = max_length
 
+    @property
+    def db_dir(self) -> str:
+        if self._db_dir is None:
+            self._db_dir = str(get_db_dir())
+        return self._db_dir
+
     @classmethod
     def from_context(
         cls,
@@ -175,7 +178,7 @@ def _build_result(self, scores: list[list[float]], labels: list[list[LabelType]]
         return build_result(np.array(scores), np.array(labels), n_classes)
 
     def clear_cache(self) -> None:
-        pass
+        self.vector_index.clear_ram()
 
     def dump(self, path: str) -> None:
         self.metadata = DNNCScorerDumpMetadata(

autointent/modules/scoring/dnnc/head_training.py

@@ -21,6 +21,7 @@
 import torch
 from sentence_transformers import CrossEncoder
 from sklearn.linear_model import LogisticRegressionCV
+from typing_extensions import Self
 
 from autointent.custom_types import LabelType
 
@@ -133,7 +134,7 @@ def set_classifier(self, clf: LogisticRegressionCV) -> None:
         self._clf = clf
 
     @classmethod
-    def load(cls, path: str) -> "CrossEncoderWithLogreg":
+    def load(cls, path: str) -> Self:
         dump_dir = Path(path)
 
         # load sklearn model
@@ -144,7 +145,7 @@ def load(cls, path: str) -> "CrossEncoderWithLogreg":
         crossencoder_dir = str(dump_dir / "crossencoder")
         model = CrossEncoder(crossencoder_dir)  # TODO control device
 
-        res = CrossEncoderWithLogreg(model)
+        res = cls(model)
         res.set_classifier(clf)
 
         return res

autointent/modules/scoring/knn/knn.py

@@ -49,16 +49,20 @@ def __init__(
             - closest: each sample has a non zero weight iff is the closest sample of some class
         - `device`: str, something like "cuda:0" or "cuda:0,1,2", a device to store embedding function
         """
-        if db_dir is None:
-            db_dir = str(get_db_dir())
         self.embedder_name = embedder_name
         self.k = k
         self.weights = weights
-        self.db_dir = db_dir
+        self._db_dir = db_dir
         self.device = device
         self.batch_size = batch_size
         self.max_length = max_length
 
+    @property
+    def db_dir(self) -> str:
+        if self._db_dir is None:
+            self._db_dir = str(get_db_dir())
+        return self._db_dir
+
     @classmethod
     def from_context(
         cls,
@@ -118,7 +122,7 @@ def predict_with_metadata(
         return scores, metadata
 
     def clear_cache(self) -> None:
-        self._vector_index.delete()
+        self._vector_index.clear_ram()
 
     def dump(self, path: str) -> None:
         self.metadata = KNNScorerDumpMetadata(