feat: add validation step (#72)

added a validation step that checks
- if the columns corresponding to the metrics passed are present
- if the columns used are of the correct datatype

Author: jjmachan

Makefile

@@ -30,4 +30,4 @@ run-benchmarks: ## Run benchmarks
 	@cd $(GIT_ROOT)/tests/benchmarks && python benchmark_eval.py
 test: ## Run tests
 	@echo "Running tests..."
-	@pytest tests/unit
+	@pytest tests/unit $(shell if [ -n "$(k)" ]; then echo "-k $(k)"; fi)

src/ragas/evaluation.py

@@ -1,29 +1,14 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
-from enum import Enum
 
 import numpy as np
 from datasets import Dataset, concatenate_datasets
 
 from ragas._analytics import EvaluationEvent, track
 from ragas.metrics.base import Metric
 from ragas.metrics.critique import AspectCritique
-
-EvaluationMode = Enum("EvaluationMode", "generative retrieval grounded")
-
-
-def get_evaluation_mode(ds: Dataset):
-    """
-    validates the dataset and returns the evaluation type
-
-    possible evaluation types
-    1. (q,a,c)
-    2. (q,a)
-    3. (q,c)
-    4. (g,a)
-    """
-    ...
+from ragas.validation import validate_column_dtypes, validate_evaluation_modes
 
 
 def evaluate(
@@ -70,16 +55,15 @@ def evaluate(
     if dataset is None:
         raise ValueError("Provide dataset!")
 
-    # TODO: validate EvaluationMode here
-    # evaluation_mode = get_evaluation_mode(dataset)
-
-    # TODO: check if all the metrics are compatible with the evaluation mode
-
     if metrics is None:
         from ragas.metrics import answer_relevancy, context_relevancy, faithfulness
 
         metrics = [answer_relevancy, context_relevancy, faithfulness]
 
+    # validation
+    validate_evaluation_modes(dataset, metrics)
+    validate_column_dtypes(dataset)
+
     # run the evaluation on dataset with different metrics
     # initialize all the models in the metrics
     [m.init_model() for m in metrics]

src/ragas/metrics/answer_relevance.py

@@ -15,7 +15,7 @@
 from transformers import AutoConfig, AutoTokenizer
 from transformers.models.auto.modeling_auto import MODEL_WITH_LM_HEAD_MAPPING_NAMES
 
-from ragas.metrics.base import Metric
+from ragas.metrics.base import EvaluationMode, Metric
 
 if t.TYPE_CHECKING:
     import numpy.typing as npt
@@ -142,6 +142,7 @@ def predict(
 @dataclass
 class AnswerRelevancy(Metric):
     name: str = "answer_relevancy"
+    evaluation_mode: EvaluationMode = EvaluationMode.qa
     batch_size: int = 32
     model_name: str = "t5-base"
 

src/ragas/metrics/base.py

@@ -9,6 +9,7 @@
 import typing as t
 from abc import ABC, abstractmethod
 from dataclasses import dataclass, field
+from enum import Enum
 from math import floor
 
 from datasets import Dataset
@@ -32,6 +33,9 @@ def make_batches(total_size: int, batch_size: int) -> list[range]:
     return batches
 
 
+EvaluationMode = Enum("EvaluationMode", "qac qa qc ga")
+
+
 @dataclass
 class Metric(ABC):
     batch_size: int
@@ -41,6 +45,11 @@ class Metric(ABC):
     def name(self) -> str:
         ...
 
+    @property
+    @abstractmethod
+    def evaluation_mode(self) -> EvaluationMode:
+        ...
+
     @abstractmethod
     def init_model():
         """

src/ragas/metrics/context_relevance.py

@@ -11,7 +11,7 @@
 from sentence_transformers import CrossEncoder
 from tqdm import tqdm
 
-from ragas.metrics.base import MetricWithLLM
+from ragas.metrics.base import EvaluationMode, MetricWithLLM
 from ragas.metrics.llms import generate
 
 CONTEXT_RELEVANCE = HumanMessagePromptTemplate.from_template(
@@ -105,6 +105,7 @@ class ContextRelevancy(MetricWithLLM):
     """
 
     name: str = "context_relavency"
+    evaluation_mode: EvaluationMode = EvaluationMode.qc
     batch_size: int = 15
     strictness: int = 2
     agreement_metric: str = "bert_score"

src/ragas/metrics/critique.py

@@ -10,7 +10,7 @@
 from langchain.prompts import ChatPromptTemplate, HumanMessagePromptTemplate
 from tqdm import tqdm
 
-from ragas.metrics.base import MetricWithLLM, _llm_factory
+from ragas.metrics.base import EvaluationMode, MetricWithLLM, _llm_factory
 from ragas.metrics.llms import generate
 
 CRITIQUE_PROMPT = HumanMessagePromptTemplate.from_template(
@@ -53,6 +53,7 @@ class AspectCritique(MetricWithLLM):
     """
 
     name: str = field(default="", repr=True)
+    evaluation_mode: EvaluationMode = EvaluationMode.qac
     definition: str = field(default="", repr=True)
     strictness: int = field(default=1, repr=False)
     batch_size: int = field(default=15, repr=False)

src/ragas/metrics/faithfulnes.py

@@ -7,7 +7,7 @@
 from langchain.prompts import ChatPromptTemplate, HumanMessagePromptTemplate
 from tqdm import tqdm
 
-from ragas.metrics.base import MetricWithLLM
+from ragas.metrics.base import EvaluationMode, MetricWithLLM
 from ragas.metrics.llms import generate
 
 if t.TYPE_CHECKING:
@@ -65,6 +65,7 @@
 @dataclass
 class Faithfulness(MetricWithLLM):
     name: str = "faithfulness"
+    evaluation_mode: EvaluationMode = EvaluationMode.qac
     batch_size: int = 15
 
     def init_model(self: t.Self):

src/ragas/validation.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from datasets import Dataset, Sequence
+
+from ragas.metrics.base import EvaluationMode, Metric
+
+
+def validate_column_dtypes(ds: Dataset):
+    for column_names in ["question", "answer"]:
+        if column_names in ds.features:
+            if ds.features[column_names].dtype != "string":
+                raise ValueError(
+                    f'Dataset feature "{column_names}" should be of type string'
+                )
+
+    for column_names in ["contexts", "ground_truths"]:
+        if column_names in ds.features:
+            if not (
+                isinstance(ds.features[column_names], Sequence)
+                and ds.features[column_names].feature.dtype == "string"
+            ):
+                raise ValueError(
+                    f'Dataset feature "{column_names}" should be of type'
+                    " Sequence[string]"
+                )
+
+
+EVALMODE_TO_COLUMNS = {
+    EvaluationMode.qac: ["question", "answer", "contexts"],
+    EvaluationMode.qa: ["question", "answer"],
+    EvaluationMode.qc: ["question", "contexts"],
+    EvaluationMode.ga: ["ground_truths", "answer"],
+}
+
+
+def validate_evaluation_modes(ds: Dataset, metrics: list[Metric]):
+    """
+    validates the dataset and returns the evaluation type
+
+    possible evaluation types
+    1. (q,a,c)
+    2. (q,a)
+    3. (q,c)
+    4. (g,a)
+    """
+
+    for m in metrics:
+        required_columns = set(EVALMODE_TO_COLUMNS[m.evaluation_mode])
+        available_columns = set(ds.features.keys())
+        if required_columns.symmetric_difference(available_columns):
+            raise ValueError(
+                f"The metric [{m.name}] that that is used requires the following "
+                f"additional columns {list(required_columns - available_columns)} "
+                "to be present in the dataset."
+            )

tests/unit/test_validation.py

@@ -0,0 +1,65 @@
+from collections import namedtuple
+
+import pytest
+from datasets import Dataset
+
+from ragas.metrics import answer_relevancy, context_relevancy, faithfulness
+from ragas.validation import validate_column_dtypes, validate_evaluation_modes
+
+CaseToTest = namedtuple(
+    "TestCase", ["q", "a", "c", "g", "is_valid_columns", "metrics", "is_valid_metrics"]
+)
+
+TEST_CASES = [
+    CaseToTest("a", "b", ["c"], None, True, [faithfulness], True),
+    CaseToTest("a", "b", ["c"], ["g"], True, [faithfulness], False),
+    CaseToTest("a", None, ["c"], None, True, [context_relevancy], True),
+    CaseToTest("a", None, "c", None, False, [context_relevancy], True),
+    CaseToTest(
+        "a", None, [["c"]], None, False, [context_relevancy, answer_relevancy], False
+    ),
+    CaseToTest("a", None, ["c"], "g", False, [context_relevancy], False),
+    CaseToTest("a", None, ["c"], [["g"]], False, [context_relevancy], False),
+    CaseToTest(1, None, ["c"], ["g"], False, [context_relevancy], False),
+    CaseToTest(1, None, None, None, False, [context_relevancy], False),
+]
+
+
+@pytest.mark.parametrize("testcase", TEST_CASES)
+def test_validate_column_dtypes(testcase):
+    dataset_dict = {}
+    if testcase.q is not None:
+        dataset_dict["question"] = [testcase.q]
+    if testcase.a is not None:
+        dataset_dict["answer"] = [testcase.a]
+    if testcase.c is not None:
+        dataset_dict["contexts"] = [testcase.c]
+    if testcase.g is not None:
+        dataset_dict["ground_truths"] = [testcase.g]
+
+    test_dataset = Dataset.from_dict(dataset_dict)
+    if testcase.is_valid_columns:
+        validate_column_dtypes(test_dataset)
+    else:
+        with pytest.raises(ValueError):
+            validate_column_dtypes(test_dataset)
+
+
+@pytest.mark.parametrize("testcase", TEST_CASES)
+def test_validate_columns_and_metrics(testcase):
+    dataset_dict = {}
+    if testcase.q is not None:
+        dataset_dict["question"] = [testcase.q]
+    if testcase.a is not None:
+        dataset_dict["answer"] = [testcase.a]
+    if testcase.c is not None:
+        dataset_dict["contexts"] = [testcase.c]
+    if testcase.g is not None:
+        dataset_dict["ground_truths"] = [testcase.g]
+    test_dataset = Dataset.from_dict(dataset_dict)
+
+    if testcase.is_valid_metrics:
+        validate_evaluation_modes(test_dataset, testcase.metrics)
+    else:
+        with pytest.raises(ValueError):
+            validate_evaluation_modes(test_dataset, testcase.metrics)