fix: raise ExceptionInRunner if executor faces any issues (#569)

Author: jjmachan

src/ragas/testset/docstore.py

@@ -15,6 +15,7 @@
 from langchain_core.pydantic_v1 import Field
 
 from ragas.embeddings.base import BaseRagasEmbeddings
+from ragas.exceptions import ExceptionInRunner
 from ragas.executor import Executor
 from ragas.run_config import RunConfig
 from ragas.testset.utils import rng
@@ -245,6 +246,9 @@ def add_nodes(
                 result_idx += 1
 
         results = executor.results()
+        if results == []:
+            raise ExceptionInRunner()
+
         for i, n in enumerate(nodes):
             if i in nodes_to_embed.keys():
                 n.embedding = results[nodes_to_embed[i]]

src/ragas/testset/evolutions.py

@@ -412,7 +412,12 @@ async def _aevolve(
         )
 
         # find a similar node and generate a question based on both
-        similar_node = self.docstore.get_similar(current_nodes.root_node)[0]
+        similar_node = self.docstore.get_similar(current_nodes.root_node)
+        if similar_node == []:
+            # retry
+            current_nodes = self.se._get_more_adjacent_nodes(current_nodes)
+            return await self.aretry_evolve(current_tries, current_nodes)
+
         prompt = self.multi_context_question_prompt.format(
             question=simple_question,
             context1=current_nodes.root_node.page_content,

src/ragas/testset/extractor.py

@@ -39,7 +39,7 @@ def save(self, cache_dir: t.Optional[str] = None) -> None:
 
 
 @dataclass
-class keyphraseExtractor(Extractor):
+class KeyphraseExtractor(Extractor):
     keyphrase_extraction_prompt: Prompt = field(
         default_factory=lambda: keyphrase_extraction_prompt
     )

src/ragas/testset/generator.py

@@ -12,6 +12,7 @@
 
 from ragas._analytics import TesetGenerationEvent, track
 from ragas.embeddings.base import BaseRagasEmbeddings, LangchainEmbeddingsWrapper
+from ragas.exceptions import ExceptionInRunner
 from ragas.executor import Executor
 from ragas.llms import BaseRagasLLM, LangchainLLMWrapper
 from ragas.run_config import RunConfig
@@ -25,7 +26,7 @@
     reasoning,
     simple,
 )
-from ragas.testset.extractor import keyphraseExtractor
+from ragas.testset.extractor import KeyphraseExtractor
 from ragas.testset.filters import EvolutionFilter, NodeFilter, QuestionFilter
 from ragas.utils import check_if_sum_is_close, is_nan
 
@@ -83,7 +84,7 @@ def with_openai(
         embeddings_model = LangchainEmbeddingsWrapper(
             OpenAIEmbeddings(model=embeddings)
         )
-        keyphrase_extractor = keyphraseExtractor(llm=generator_llm_model)
+        keyphrase_extractor = KeyphraseExtractor(llm=generator_llm_model)
         if docstore is None:
             from langchain.text_splitter import TokenTextSplitter
 
@@ -241,6 +242,9 @@ def generate(
 
         try:
             test_data_rows = exec.results()
+            if test_data_rows == []:
+                raise ExceptionInRunner()
+
         except ValueError as e:
             raise e
         # make sure to ignore any NaNs that might have been returned