fix: answer_correctness doesn't reset stuff properly (#562)

Author: jjmachan

docs/howtos/integrations/athina.ipynb

@@ -47,18 +47,23 @@
    "outputs": [],
    "source": [
     "import os\n",
-    "from athina.evals import RagasAnswerCorrectness, RagasAnswerRelevancy, RagasContextRelevancy, RagasFaithfulness \n",
+    "from athina.evals import (\n",
+    "    RagasAnswerCorrectness,\n",
+    "    RagasAnswerRelevancy,\n",
+    "    RagasContextRelevancy,\n",
+    "    RagasFaithfulness,\n",
+    ")\n",
     "from athina.loaders import RagasLoader\n",
     "from athina.keys import AthinaApiKey, OpenAiApiKey\n",
     "from athina.runner.run import EvalRunner\n",
     "import pandas as pd\n",
     "\n",
     "# Set your API keys\n",
-    "OpenAiApiKey.set_key(os.getenv('OPENAI_API_KEY'))\n",
-    "AthinaApiKey.set_key(os.getenv('ATHINA_API_KEY'))\n",
+    "OpenAiApiKey.set_key(os.getenv(\"OPENAI_API_KEY\"))\n",
+    "AthinaApiKey.set_key(os.getenv(\"ATHINA_API_KEY\"))\n",
     "\n",
     "# Load your dataset from a dictionary, json, or csv: https://docs.athina.ai/evals/loading_data\n",
-    "dataset = RagasLoader().load_json('raw_data.json')\n",
+    "dataset = RagasLoader().load_json(\"raw_data.json\")\n",
     "\n",
     "# Configure the eval suite\n",
     "eval_model = \"gpt-3.5-turbo\"\n",
@@ -73,7 +78,7 @@
     "batch_eval_result = EvalRunner.run_suite(\n",
     "    evals=eval_suite,\n",
     "    data=dataset,\n",
-    "    max_parallel_evals=1,   # If you increase this, you may run into rate limits\n",
+    "    max_parallel_evals=1,  # If you increase this, you may run into rate limits\n",
     ")\n",
     "\n",
     "pd.DataFrame(batch_eval_result)"

src/ragas/evaluation.py

@@ -5,18 +5,19 @@
 
 import numpy as np
 from datasets import Dataset, concatenate_datasets
-from langchain_core.language_models import BaseLanguageModel as LangchainLLM
 from langchain_core.embeddings import Embeddings as LangchainEmbeddings
+from langchain_core.language_models import BaseLanguageModel as LangchainLLM
 
 from ragas._analytics import EvaluationEvent, track
 from ragas.callbacks import new_group
 from ragas.embeddings.base import BaseRagasEmbeddings, LangchainEmbeddingsWrapper
+from ragas.exceptions import ExceptionInRunner
 from ragas.executor import Executor
 from ragas.llms.base import BaseRagasLLM, LangchainLLMWrapper
+from ragas.metrics._answer_correctness import AnswerCorrectness
 from ragas.metrics.base import Metric, MetricWithEmbeddings, MetricWithLLM
 from ragas.metrics.critique import AspectCritique
 from ragas.run_config import RunConfig
-from ragas.exceptions import ExceptionInRunner
 
 # from ragas.metrics.critique import AspectCritique
 from ragas.validation import (
@@ -158,6 +159,7 @@ def evaluate(
     binary_metrics = []
     llm_changed: t.List[int] = []
     embeddings_changed: t.List[int] = []
+    answer_correctness_is_set = -1
     for i, metric in enumerate(metrics):
         if isinstance(metric, AspectCritique):
             binary_metrics.append(metric.name)
@@ -169,6 +171,9 @@ def evaluate(
             if metric.embeddings is None:
                 metric.embeddings = embeddings
                 embeddings_changed.append(i)
+        if isinstance(metric, AnswerCorrectness):
+            if metric.answer_similarity is None:
+                answer_correctness_is_set = i
 
     # initialize all the models in the metrics
     [m.init(run_config) for m in metrics]
@@ -237,6 +242,10 @@ def evaluate(
             t.cast(MetricWithLLM, metrics[i]).llm = None
         for i in embeddings_changed:
             t.cast(MetricWithEmbeddings, metrics[i]).embeddings = None
+        if answer_correctness_is_set != -1:
+            t.cast(
+                AnswerCorrectness, metrics[answer_correctness_is_set]
+            ).answer_similarity = None
 
     # log the evaluation event
     metrics_names = [m.name for m in metrics]

src/ragas/executor.py

@@ -2,9 +2,9 @@
 
 import asyncio
 import logging
+import threading
 import typing as t
 from dataclasses import dataclass, field
-import threading
 
 import numpy as np
 from tqdm.auto import tqdm

src/ragas/llms/prompt.py

@@ -149,7 +149,6 @@ def format(self, **kwargs: t.Any) -> PromptValue:
     def adapt(
         self, language: str, llm: BaseRagasLLM, cache_dir: t.Optional[str] = None
     ) -> Prompt:
-      
         def get_all_keys(nested_json):
             keys = set()
             for key, value in nested_json.items():
@@ -160,7 +159,7 @@ def get_all_keys(nested_json):
 
         if self.language == language:
             return self
-          
+
         # TODO: Add callbacks
         cache_dir = cache_dir if cache_dir else get_cache_dir()
         if os.path.exists(os.path.join(cache_dir, language, f"{self.name}.json")):