feat: added ragas_score and fixed dependency and ci issues (#36)

* ci fixes

* add src directory file structure

* update pyproject

* fix _version.py

* fix linting and type issues

* fix dependencies

* fix annotation bug

* added ragas score

Author: jjmachan

.gitignore

@@ -164,3 +164,4 @@ ragas/_version.py
 experiments/**/data
 experiments/**/storage
 **/fil-result/
+src/ragas/_version.py

Makefile

@@ -8,17 +8,17 @@ format: ## Running code formatter: black and isort
 	@echo "(isort) Ordering imports..."
 	@isort .
 	@echo "(black) Formatting codebase..."
-	@black --config pyproject.toml ragas tests examples
+	@black --config pyproject.toml src tests examples experiments
 	@echo "(black) Formatting stubs..."
-	@find ragas -name "*.pyi" ! -name "*_pb2*" -exec black --pyi --config pyproject.toml {} \;
+	@find src -name "*.pyi" ! -name "*_pb2*" -exec black --pyi --config pyproject.toml {} \;
 	@echo "(ruff) Running fix only..."
-	@ruff check ragas examples tests --fix-only
+	@ruff check src examples tests --fix-only
 lint: ## Running lint checker: ruff
 	@echo "(ruff) Linting development project..."
-	@ruff check ragas examples tests
+	@ruff check src examples tests
 type: ## Running type checker: pyright
 	@echo "(pyright) Typechecking codebase..."
-	@pyright ragas
+	@pyright src
 clean: ## Clean all generated files
 	@echo "Cleaning all generated files..."
 	@cd $(GIT_ROOT)/docs && make clean

pyproject.toml

@@ -1,23 +1,25 @@
 [project]
 name = "ragas"
 dependencies = [
-    "Levenshtein",
-    "rouge-score",
     "numpy",
     "transformers",
     "sentence-transformers",
-    "nltk",
     "datasets",
-    "spacy<4.0.0,>=3.0.0",
     "protobuf<=3.20.0",
+    "backoff",
+    "openai",
 ]
 dynamic = ["version", "readme"]
 
+[tool.setuptools]
+package-dir = {"" = "src"}
+
 [tool.setuptools.dynamic]
 readme = {file = ["README.md"], content-type = "text/plain"}
 
 [build-system]
 requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"]
 build-backend = "setuptools.build_meta"
+
 [tool.setuptools_scm]
-write_to = "ragas/_version.py"
+write_to = "src/ragas/_version.py"

ragas/__init__.py renamed to src/ragas/__init__.py

@@ -1,12 +1,10 @@
 from __future__ import annotations
 
-import typing as t
 from dataclasses import dataclass
 from enum import Enum
 
 import numpy as np
 from datasets import Dataset, concatenate_datasets
-from tqdm import tqdm
 
 from ragas.metrics.base import Metric
 
@@ -44,19 +42,24 @@ def evaluate(
     [m.init_model() for m in metrics]
 
     scores = []
-    for metric in tqdm(metrics):
+    for metric in metrics:
         scores.append(metric.score(dataset).select_columns(metric.name))
 
-    return Result(concatenate_datasets(scores))
+    return Result(concatenate_datasets(scores, axis=1))
 
 
 @dataclass
 class Result(dict):
     scores: Dataset
 
     def __post_init__(self):
+        values = []
         for cn in self.scores.column_names:
-            self[cn] = np.mean(self.scores[cn])
+            value = np.mean(self.scores[cn])
+            self[cn] = value
+            values.append(value)
+
+        self["ragas_score"] = len(values) / np.sum(1.0 / np.array(values))
 
     def describe(self):
         description = {}

ragas/evaluation.py renamed to src/ragas/evaluation.py

@@ -121,7 +121,7 @@ def predict(
     ) -> npt.NDArray[np.float64]:
         predictions = []
         dataloader = DataLoader(
-            sentences, batch_size=batch_size, collate_fn=self.collate_fn
+            sentences, batch_size=batch_size, collate_fn=self.collate_fn  # type: ignore
         )
 
         if show_progress:

ragas/exceptions.py renamed to src/ragas/exceptions.py

@@ -47,8 +47,11 @@ def get_batches(self, dataset_size: int):
             range(i, i + self.batch_size)
             for i in range(0, self.batch_size * num_batches, self.batch_size)
         ]
-        batches.append(
-            range(self.batch_size * num_batches, self.batch_size * num_batches + tail)
-        )
+        if tail != 0:
+            batches.append(
+                range(
+                    self.batch_size * num_batches, self.batch_size * num_batches + tail
+                )
+            )
 
         return batches