renaming ground_truths to ground_truth. (#640)

alexjyong · web-flow · commit 2a3c3abdbc68 · 2024-02-23T00:46:31.000-08:00
## Description
Noticed that there were still references to `ground_truths` in the code
and documentation (barring the validation function of course) after we
updated this to `ground_truth`. This PR cleans that part up.

Also updated the raise_exceptions error message to read
`raise_exceptions=False` instead of `raise_exception=False`
diff --git a/README.md b/README.md
@@ -71,7 +71,7 @@ os.environ["OPENAI_API_KEY"] = "your-openai-key"
 
 # prepare your huggingface dataset in the format
 # Dataset({
-#     features: ['question', 'contexts', 'answer', 'ground_truths'],
+#     features: ['question', 'contexts', 'answer', 'ground_truth'],
 #     num_rows: 25
 # })
 
diff --git a/docs/concepts/metrics/answer_correctness.md b/docs/concepts/metrics/answer_correctness.md
@@ -24,7 +24,7 @@ answer_correctness = AnswerCorrectness(
 )
 
 # Dataset({
-#     features: ['answer','ground_truths'],
+#     features: ['answer','ground_truth'],
 #     num_rows: 25
 # })
 dataset: Dataset
diff --git a/docs/concepts/metrics/context_recall.md b/docs/concepts/metrics/context_recall.md
@@ -37,7 +37,7 @@ context_recall = ContextRecall(
 
 )
 # Dataset({
-#     features: ['contexts','ground_truths'],
+#     features: ['contexts','ground_truth'],
 #     num_rows: 25
 # })
 dataset: Dataset
diff --git a/docs/concepts/metrics/semantic_similarity.md b/docs/concepts/metrics/semantic_similarity.md
@@ -24,7 +24,7 @@ answer_similarity = AnswerSimilarity()
 
 
 # Dataset({
-#     features: ['answer','ground_truths'],
+#     features: ['answer','ground_truth'],
 #     num_rows: 25
 # })
 dataset: Dataset
diff --git a/docs/howtos/applications/compare_llms.md b/docs/howtos/applications/compare_llms.md
@@ -110,7 +110,7 @@ def generate_responses(query_engine, test_questions, test_answers):
         "contexts": contexts,
   }
   if test_answers is not None:
-    dataset_dict["ground_truths"] = test_answers
+    dataset_dict["ground_truth"] = test_answers
   ds = Dataset.from_dict(dataset_dict)
   return ds
 ```
@@ -214,4 +214,4 @@ analysis(
 ### Score distribution analysis
 <p align="left">
 <img src="../../_static/imgs/compare-llm-result.png" alt="results" width="800" height="600" />
-</p>
+</p>
diff --git a/docs/howtos/applications/custom_prompts.md b/docs/howtos/applications/custom_prompts.md
@@ -17,7 +17,7 @@ amnesty_dataset
 ```{code-block} bash
 DatasetDict({
     train: Dataset({
-        features: ['question', 'ground_truths', 'answer', 'contexts'],
+        features: ['question', 'ground_truth', 'answer', 'contexts'],
         num_rows: 20
     })
 })
@@ -103,4 +103,4 @@ evaluating with [faithfulness]
 100%|██████████| 1/1 [02:31<00:00, 151.79s/it]
 
 {'faithfulness': 0.7879}
-```
+```
diff --git a/docs/howtos/applications/data_preparation.md b/docs/howtos/applications/data_preparation.md
@@ -24,7 +24,7 @@ data_samples = {
     'answer': ['The first superbowl was held on January 15, 1967', 'The most super bowls have been won by The New England Patriots'],
     'contexts' : [['The Super Bowl....season since 1966,','replacing the NFL...in February.'], 
     ['The Green Bay Packers...Green Bay, Wisconsin.','The Packers compete...Football Conference']],
-    'ground_truths': [['The first superbowl was held on January 15, 1967'], ['The New England Patriots have won the Super Bowl a record six times']]
+    'ground_truth': [['The first superbowl was held on January 15, 1967'], ['The New England Patriots have won the Super Bowl a record six times']]
 }
 dataset = Dataset.from_dict(data_samples)
 ```
diff --git a/docs/howtos/applications/use_prompt_adaptation.md b/docs/howtos/applications/use_prompt_adaptation.md
@@ -21,7 +21,7 @@ hindi_dataset
 ```{code-block}
 DatasetDict({
     train: Dataset({
-        features: ['question', 'ground_truths', 'answer', 'contexts'],
+        features: ['question', 'ground_truth', 'answer', 'contexts'],
         num_rows: 20
     })
 })
@@ -153,4 +153,4 @@ distributions = {
 # generate testset
 testset = generator.generate_with_langchain_docs(documents, 10,distributions,with_debugging_logs=True)
 testset.to_pandas()
-```
+```
diff --git a/docs/howtos/customisations/aws-bedrock.ipynb b/docs/howtos/customisations/aws-bedrock.ipynb
@@ -163,7 +163,7 @@
       "text/plain": [
        "DatasetDict({\n",
        "    eval: Dataset({\n",
-       "        features: ['question', 'ground_truths', 'answer', 'contexts'],\n",
+       "        features: ['question', 'ground_truth', 'answer', 'contexts'],\n",
        "        num_rows: 20\n",
        "    })\n",
        "})"
@@ -257,7 +257,7 @@
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>question</th>\n",
-       "      <th>ground_truths</th>\n",
+       "      <th>ground_truth</th>\n",
        "      <th>answer</th>\n",
        "      <th>contexts</th>\n",
        "      <th>faithfulness</th>\n",
@@ -310,7 +310,7 @@
        "1  Which companies are the main contributors to G...   \n",
        "2  Which private companies in the Americas are th...   \n",
        "\n",
-       "                                       ground_truths  \\\n",
+       "                                       ground_truth  \\\n",
        "0  [The global implications of the USA Supreme Co...   \n",
        "1  [According to the Carbon Majors database, the ...   \n",
        "2  [The largest private companies in the Americas...   \n",
diff --git a/docs/howtos/customisations/azure-openai.ipynb b/docs/howtos/customisations/azure-openai.ipynb
@@ -60,7 +60,7 @@
       "text/plain": [
        "DatasetDict({\n",
        "    baseline: Dataset({\n",
-       "        features: ['question', 'ground_truths', 'answer', 'contexts'],\n",
+       "        features: ['question', 'ground_truth', 'answer', 'contexts'],\n",
        "        num_rows: 30\n",
        "    })\n",
        "})"
@@ -285,7 +285,7 @@
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>question</th>\n",
-       "      <th>ground_truths</th>\n",
+       "      <th>ground_truth</th>\n",
        "      <th>answer</th>\n",
        "      <th>contexts</th>\n",
        "      <th>faithfulness</th>\n",
@@ -368,7 +368,7 @@
        "3         Applying for and receiving business credit   \n",
        "4               401k Transfer After Business Closure   \n",
        "\n",
-       "                                       ground_truths  \\\n",
+       "                                       ground_truth  \\\n",
        "0  [Have the check reissued to the proper payee.J...   \n",
        "1  [Sure you can.  You can fill in whatever you w...   \n",
        "2  [You're confusing a lot of things here. Compan...   \n",
diff --git a/docs/howtos/customisations/gcp-vertexai.ipynb b/docs/howtos/customisations/gcp-vertexai.ipynb
@@ -64,7 +64,7 @@
       "text/plain": [
        "DatasetDict({\n",
        "    baseline: Dataset({\n",
-       "        features: ['question', 'ground_truths', 'answer', 'contexts'],\n",
+       "        features: ['question', 'ground_truth', 'answer', 'contexts'],\n",
        "        num_rows: 30\n",
        "    })\n",
        "})"
@@ -310,7 +310,7 @@
        "      <th>question</th>\n",
        "      <th>contexts</th>\n",
        "      <th>answer</th>\n",
-       "      <th>ground_truths</th>\n",
+       "      <th>ground_truth</th>\n",
        "      <th>faithfulness</th>\n",
        "      <th>answer_relevancy</th>\n",
        "      <th>context_recall</th>\n",
@@ -345,7 +345,7 @@
        "                                              answer  \\\n",
        "0  \\nThe best way to deposit a cheque issued to a...   \n",
        "\n",
-       "                                       ground_truths  faithfulness  \\\n",
+       "                                       ground_truth  faithfulness  \\\n",
        "0  [Have the check reissued to the proper payee.J...           1.0   \n",
        "\n",
        "   answer_relevancy  context_recall  context_precision  harmfulness  \n",
diff --git a/docs/howtos/integrations/langchain.ipynb b/docs/howtos/integrations/langchain.ipynb
@@ -126,7 +126,7 @@
     "]\n",
     "\n",
     "examples = [\n",
-    "    {\"query\": q, \"ground_truths\": [eval_answers[i]]}\n",
+    "    {\"query\": q, \"ground_truth\": [eval_answers[i]]}\n",
     "    for i, q in enumerate(eval_questions)\n",
     "]"
    ]
@@ -381,7 +381,7 @@
     "    for e in examples:\n",
     "        client.create_example(\n",
     "            inputs={\"query\": e[\"query\"]},\n",
-    "            outputs={\"ground_truths\": e[\"ground_truths\"]},\n",
+    "            outputs={\"ground_truth\": e[\"ground_truth\"]},\n",
     "            dataset_id=dataset.id,\n",
     "        )\n",
     "\n",
diff --git a/docs/howtos/integrations/langfuse.ipynb b/docs/howtos/integrations/langfuse.ipynb
@@ -61,7 +61,7 @@
     "- `question`: list[str] - These are the questions your RAG pipeline will be evaluated on.\n",
     "- `answer`: list[str] - The answer generated from the RAG pipeline and given to the user.\n",
     "- `contexts`: list[list[str]] - The contexts which were passed into the LLM to answer the question.\n",
-    "- `ground_truths`: list[list[str]] - The ground truth answer to the questions. However, this can be ignored for online evaluations since we will not have access to ground-truth data in our case."
+    "- `ground_truth`: list[list[str]] - The ground truth answer to the questions. However, this can be ignored for online evaluations since we will not have access to ground-truth data in our case."
    ]
   },
   {
@@ -74,7 +74,7 @@
      "data": {
       "text/plain": [
        "Dataset({\n",
-       "    features: ['question', 'ground_truths', 'answer', 'contexts'],\n",
+       "    features: ['question', 'ground_truth', 'answer', 'contexts'],\n",
        "    num_rows: 30\n",
        "})"
       ]
diff --git a/docs/howtos/integrations/llamaindex.ipynb b/docs/howtos/integrations/llamaindex.ipynb
@@ -329,7 +329,7 @@
        "      <th>question</th>\n",
        "      <th>contexts</th>\n",
        "      <th>answer</th>\n",
-       "      <th>ground_truths</th>\n",
+       "      <th>ground_truth</th>\n",
        "      <th>faithfulness</th>\n",
        "      <th>answer_relevancy</th>\n",
        "      <th>context_precision</th>\n",
@@ -424,7 +424,7 @@
        "3  \\nNew York City was named in honor of the Duke...   \n",
        "4  \\nThe Statue of Liberty is a symbol of the Uni...   \n",
        "\n",
-       "                                       ground_truths  faithfulness  \\\n",
+       "                                       ground_truth  faithfulness  \\\n",
        "0                                        [8,804,000]           1.0   \n",
        "1                                           [Queens]           0.0   \n",
        "2  [New York City's economic significance is vast...           1.0   \n",
diff --git a/docs/howtos/integrations/zeno.ipynb b/docs/howtos/integrations/zeno.ipynb
@@ -113,7 +113,7 @@
     "            \"type\": \"vstack\",\n",
     "            \"keys\": {\n",
     "                \"answer\": {\"type\": \"markdown\"},\n",
-    "                \"ground_truths\": {\n",
+    "                \"ground_truth\": {\n",
     "                    \"type\": \"list\",\n",
     "                    \"elements\": {\"type\": \"markdown\"},\n",
     "                    \"border\": True,\n",
@@ -152,7 +152,7 @@
     "        \"data\": df.apply(\n",
     "            lambda x: {\"question\": x[\"question\"], \"texts\": list(x[\"contexts\"])}, axis=1\n",
     "        ),\n",
-    "        \"label\": df[\"ground_truths\"].apply(lambda x: \"\\n\".join(x)),\n",
+    "        \"label\": df[\"ground_truth\"].apply(lambda x: \"\\n\".join(x)),\n",
     "    }\n",
     ")\n",
     "data_df[\"id\"] = data_df.index\n",
@@ -187,7 +187,7 @@
     "].copy()\n",
     "\n",
     "output_df[\"output\"] = df.apply(\n",
-    "    lambda x: {\"answer\": x[\"answer\"], \"ground_truths\": list(x[\"ground_truths\"])}, axis=1\n",
+    "    lambda x: {\"answer\": x[\"answer\"], \"ground_truth\": list(x[\"ground_truth\"])}, axis=1\n",
     ")\n",
     "output_df[\"id\"] = output_df.index\n",
     "\n",
diff --git a/src/ragas/evaluation.py b/src/ragas/evaluation.py
@@ -50,7 +50,7 @@ def evaluate(
 
     Parameters
     ----------
-    dataset : Dataset[question: list[str], contexts: list[list[str]], answer: list[str], ground_truths: list[list[str]]]
+    dataset : Dataset[question: list[str], contexts: list[list[str]], answer: list[str], ground_truth: list[list[str]]]
         The dataset in the format of ragas which the metrics will use to score the RAG
         pipeline with
     metrics : list[Metric] , optional
@@ -109,7 +109,7 @@ def evaluate(
 
     >>> dataset
     Dataset({
-        features: ['question', 'ground_truths', 'answer', 'contexts'],
+        features: ['question', 'ground_truth', 'answer', 'contexts'],
         num_rows: 30
     })
 
diff --git a/src/ragas/exceptions.py b/src/ragas/exceptions.py
@@ -33,5 +33,5 @@ class ExceptionInRunner(RagasException):
     """
 
     def __init__(self):
-        msg = "The runner thread which was running the jobs raised an exeception. Read the traceback above to debug it. You can also pass `raise_exception=False` incase you want to show only a warning message instead."
+        msg = "The runner thread which was running the jobs raised an exeception. Read the traceback above to debug it. You can also pass `raise_exceptions=False` incase you want to show only a warning message instead."
         super().__init__(msg)
diff --git a/src/ragas/metrics/base.py b/src/ragas/metrics/base.py
@@ -2,7 +2,7 @@
 Q - question
 A - answer: generated_text from RAG pipeline
 C - contexts: context used for generation
-G - ground_truths: ground truth answer
+G - ground_truth: ground truth answer
 """
 from __future__ import annotations
 

Original file line number	Diff line number	Diff line change
`@@ -24,7 +24,7 @@ answer_correctness = AnswerCorrectness(`
`24`	`24`	`)`
`25`	`25`
`26`	`26`	`# Dataset({`
`27`		`-# features: ['answer','ground_truths'],`
	`27`	`+# features: ['answer','ground_truth'],`
`28`	`28`	`# num_rows: 25`
`29`	`29`	`# })`
`30`	`30`	`dataset: Dataset`
Original file line number	Diff line number	Diff line change
`@@ -37,7 +37,7 @@ context_recall = ContextRecall(`
`37`	`37`
`38`	`38`	`)`
`39`	`39`	`# Dataset({`
`40`		`-# features: ['contexts','ground_truths'],`
	`40`	`+# features: ['contexts','ground_truth'],`
`41`	`41`	`# num_rows: 25`
`42`	`42`	`# })`
`43`	`43`	`dataset: Dataset`
Original file line number	Diff line number	Diff line change
`@@ -24,7 +24,7 @@ data_samples = {`
`24`	`24`	`'answer': ['The first superbowl was held on January 15, 1967', 'The most super bowls have been won by The New England Patriots'],`
`25`	`25`	`'contexts' : [['The Super Bowl....season since 1966,','replacing the NFL...in February.'],`
`26`	`26`	`['The Green Bay Packers...Green Bay, Wisconsin.','The Packers compete...Football Conference']],`
`27`		`- 'ground_truths': [['The first superbowl was held on January 15, 1967'], ['The New England Patriots have won the Super Bowl a record six times']]`
	`27`	`+ 'ground_truth': [['The first superbowl was held on January 15, 1967'], ['The New England Patriots have won the Super Bowl a record six times']]`
`28`	`28`	`}`
`29`	`29`	`dataset = Dataset.from_dict(data_samples)`
`30`	`30`	```