fix: sentence segmenter for multi languages (#946)

#947 
There was a bug when using pysdb sentence segmenter with multiple
languages other than English, this PR fixes it.

Author: shahules786

src/ragas/metrics/_answer_correctness.py

@@ -6,7 +6,6 @@
 
 import numpy as np
 from langchain_core.pydantic_v1 import BaseModel
-from pysbd import Segmenter
 
 from ragas.llms.output_parser import RagasoutputParser, get_json_format_instructions
 from ragas.llms.prompt import Prompt, PromptValue
@@ -16,7 +15,12 @@
     HasSegmentMethod,
     _statements_output_parser,
 )
-from ragas.metrics.base import EvaluationMode, MetricWithEmbeddings, MetricWithLLM
+from ragas.metrics.base import (
+    EvaluationMode,
+    MetricWithEmbeddings,
+    MetricWithLLM,
+    get_segmenter,
+)
 from ragas.run_config import RunConfig
 
 if t.TYPE_CHECKING:
@@ -176,7 +180,7 @@ def __post_init__(self: t.Self):
 
         if self.sentence_segmenter is None:
             language = self.long_form_answer_prompt.language
-            self.sentence_segmenter = Segmenter(language=language, clean=False)
+            self.sentence_segmenter = get_segmenter(language=language, clean=False)
 
     def init(self, run_config: RunConfig):
         super().init(run_config)

src/ragas/metrics/_faithfulness.py

@@ -7,11 +7,10 @@
 
 import numpy as np
 from langchain_core.pydantic_v1 import BaseModel, Field
-from pysbd import Segmenter
 
 from ragas.llms.output_parser import RagasoutputParser, get_json_format_instructions
 from ragas.llms.prompt import Prompt
-from ragas.metrics.base import EvaluationMode, MetricWithLLM, ensembler
+from ragas.metrics.base import EvaluationMode, MetricWithLLM, ensembler, get_segmenter
 
 if t.TYPE_CHECKING:
     from langchain_core.callbacks import Callbacks
@@ -81,7 +80,7 @@ def dicts(self) -> t.List[t.Dict]:
     ],
     input_keys=["question", "answer", "sentences"],
     output_key="analysis",
-    language="en",
+    language="english",
 )
 
 
@@ -160,7 +159,7 @@ def dicts(self) -> t.List[t.Dict]:
     input_keys=["context", "statements"],
     output_key="answer",
     output_type="json",
-    language="en",
+    language="english",
 )  # noqa: E501
 
 
@@ -190,7 +189,7 @@ def reproducibility(self, value):
     def __post_init__(self):
         if self.sentence_segmenter is None:
             language = self.nli_statements_message.language
-            self.sentence_segmenter = Segmenter(language=language, clean=False)
+            self.sentence_segmenter = get_segmenter(language=language, clean=False)
 
     def _create_nli_prompt(self, row: t.Dict, statements: t.List[str]) -> PromptValue:
         assert self.llm is not None, "llm must be set to compute score"

src/ragas/metrics/base.py

@@ -23,6 +23,10 @@
     from ragas.embeddings import BaseRagasEmbeddings
     from ragas.llms import BaseRagasLLM
 
+from pysbd import Segmenter
+from pysbd.languages import LANGUAGE_CODES
+
+LANGUAGE_CODES = {v.__name__.lower(): k for k, v in LANGUAGE_CODES.items()}
 
 EvaluationMode = Enum("EvaluationMode", "qac qa qc gc ga qga qcg")
 
@@ -191,5 +195,20 @@ def from_discrete(self, inputs: list[list[t.Dict]], attribute: str):
         return verdict_agg
 
 
-ensembler = Ensember()
+def get_segmenter(
+    language: str = "english", clean: bool = False, char_span: bool = False
+):
+    """
+    Get a sentence segmenter for a given language
+    """
+    language = language.lower()
+    if language not in LANGUAGE_CODES:
+        raise ValueError(
+            f"Language '{language}' not supported. Supported languages: {LANGUAGE_CODES.keys()}"
+        )
+    return Segmenter(
+        language=LANGUAGE_CODES[language], clean=clean, char_span=char_span
+    )
+
 
+ensembler = Ensember()