fix: proper error message and fixed hf embedding error (#561)

Author: jjmachan

src/ragas/embeddings/__init__.py

@@ -1,6 +1,11 @@
-from ragas.embeddings.base import BaseRagasEmbeddings, HuggingfaceEmbeddings
+from ragas.embeddings.base import (
+    BaseRagasEmbeddings,
+    HuggingfaceEmbeddings,
+    LangchainEmbeddingsWrapper,
+)
 
 __all__ = [
     "HuggingfaceEmbeddings",
     "BaseRagasEmbeddings",
+    "LangchainEmbeddingsWrapper",
 ]

src/ragas/evaluation.py

@@ -16,6 +16,7 @@
 from ragas.metrics.base import Metric, MetricWithEmbeddings, MetricWithLLM
 from ragas.metrics.critique import AspectCritique
 from ragas.run_config import RunConfig
+from ragas.exceptions import ExceptionInRunner
 
 # from ragas.metrics.critique import AspectCritique
 from ragas.validation import (
@@ -202,6 +203,8 @@ def evaluate(
     try:
         # get the results
         results = executor.results()
+        if results == []:
+            raise ExceptionInRunner()
 
         # convert results to dataset_like
         for i, _ in enumerate(dataset):

src/ragas/exceptions.py

@@ -25,3 +25,13 @@ def __init__(self, evolution: Evolution):
         self.evolution = evolution
         msg = f"Max retries exceeded for evolution {evolution.__class__.__name__}."
         super().__init__(msg)
+
+
+class ExceptionInRunner(RagasException):
+    """
+    Exception raised when an exception is raised in the executor.
+    """
+
+    def __init__(self):
+        msg = "The runner thread which was running the jobs raised an exeception. Read the traceback above to debug it. You can also pass `raise_exception=False` incase you want to show only a warning message instead."
+        super().__init__(msg)

src/ragas/executor.py

@@ -4,7 +4,7 @@
 import logging
 import typing as t
 from dataclasses import dataclass, field
-from threading import Thread
+import threading
 
 import numpy as np
 from tqdm.auto import tqdm
@@ -14,7 +14,16 @@
 logger = logging.getLogger(__name__)
 
 
-class Runner(Thread):
+def runner_exception_hook(args: threading.ExceptHookArgs):
+    print(args)
+    raise args.exc_type
+
+
+# set a custom exception hook
+# threading.excepthook = runner_exception_hook
+
+
+class Runner(threading.Thread):
     def __init__(
         self,
         jobs: t.List[t.Tuple[t.Coroutine, str]],

src/ragas/metrics/_answer_similarity.py

@@ -54,32 +54,29 @@ async def _ascore(
     ) -> float:
         assert self.embeddings is not None, "embeddings must be set"
 
-        ground_truth, answers = row["ground_truth"], row["answer"]
-        ground_truth = [item[0] for item in ground_truth]
+        ground_truth = t.cast(str, row["ground_truth"])
+        answer = t.cast(str, row["answer"])
 
         if self.is_cross_encoder and isinstance(self.embeddings, HuggingfaceEmbeddings):
             raise NotImplementedError(
                 "async score [ascore()] not implemented for HuggingFace embeddings"
             )
         else:
-            embeddings_1 = np.array(await self.embeddings.embed_texts(ground_truth))
-            embeddings_2 = np.array(await self.embeddings.embed_texts(answers))
+            embedding_1 = np.array(await self.embeddings.embed_text(ground_truth))
+            embedding_2 = np.array(await self.embeddings.embed_text(answer))
             # Normalization factors of the above embeddings
-            norms_1 = np.linalg.norm(embeddings_1, axis=1, keepdims=True)
-            norms_2 = np.linalg.norm(embeddings_2, axis=1, keepdims=True)
-            embeddings_1_normalized = embeddings_1 / norms_1
-            embeddings_2_normalized = embeddings_2 / norms_2
-            similarity = embeddings_1_normalized @ embeddings_2_normalized.T
-            if similarity.size == 1:
-                scores = similarity.flatten()
-            else:
-                scores = np.diagonal(similarity)
-
-        assert isinstance(scores, np.ndarray), "Expects ndarray"
+            norms_1 = np.linalg.norm(embedding_1, keepdims=True)
+            norms_2 = np.linalg.norm(embedding_2, keepdims=True)
+            embedding_1_normalized = embedding_1 / norms_1
+            embedding_2_normalized = embedding_2 / norms_2
+            similarity = embedding_1_normalized @ embedding_2_normalized.T
+            score = similarity.flatten()
+
+        assert isinstance(score, np.ndarray), "Expects ndarray"
         if self.threshold:
-            scores = scores >= self.threshold
+            score = score >= self.threshold
 
-        return scores.tolist()[0]
+        return score.tolist()[0]
 
 
 answer_similarity = AnswerSimilarity()