fix: error in handling device for tensors (#61)

fixes #60

Author: jjmachan

src/ragas/evaluation.py

@@ -85,6 +85,7 @@ def evaluate(
 
     scores = []
     for metric in metrics:
+        print(f"evaluating with [{metric.name}]")
         scores.append(metric.score(dataset).select_columns(metric.name))
 
     # log the evaluation event

src/ragas/metrics/answer_relevance.py

@@ -129,6 +129,8 @@ def predict(
 
         for _, data in enumerate(dataloader):
             inputs, labels = data
+            inputs = {k: v.to(self._target_device) for k, v in inputs.items()}
+            labels = labels.to(self._target_device)
             with torch.no_grad():
                 logits = self.model(**inputs, output_hidden_states=False).logits
                 loss = self.get_loss(logits, labels)

tests/benchmarks/benchmark_eval.py

@@ -1,6 +1,6 @@
 import os
 
-from datasets import Dataset
+from datasets import Dataset, load_dataset
 from torch.cuda import is_available
 
 from ragas import evaluate
@@ -9,9 +9,13 @@
 DEVICE = "cuda" if is_available() else "cpu"
 
 PATH_TO_DATSET_GIT_REPO = "../../../datasets/fiqa/"
-assert os.path.isdir(PATH_TO_DATSET_GIT_REPO), "Dataset not found"
-ds = Dataset.from_json(os.path.join(PATH_TO_DATSET_GIT_REPO, "gen_ds.json"))
-assert isinstance(ds, Dataset)
+dataset_dir = os.environ.get("DATASET_DIR", PATH_TO_DATSET_GIT_REPO)
+if os.path.isdir(dataset_dir):
+    ds = Dataset.from_csv(os.path.join(dataset_dir, "baseline.csv"))
+    assert isinstance(ds, Dataset)
+else:
+    # data
+    ds = load_dataset("explodinggradients/fiqa", "ragas_eval")["baseline"]
 
 if __name__ == "__main__":
     result = evaluate(