feat: add option to change regex to parse score (#237)

Author: kevin3314

flexeval/core/metric/llm_score.py

@@ -14,12 +14,14 @@
 from .utils import extract_text_from_outputs, validate_inputs
 
 
-def parse_score_from_evaluator_output(evaluator_output: str, valid_score_range: tuple[int, int] | None) -> int | None:
+def parse_score_from_evaluator_output(
+    evaluator_output: str, valid_score_range: tuple[int, int] | None, regex_to_parse_score: str = r"(\d+)"
+) -> int | None:
     """Extract the last integer value from the evaluator output.
 
     Return None if parsing fails.
     """
-    matched = re.findall(r"(\d+)", evaluator_output)
+    matched = re.findall(regex_to_parse_score, evaluator_output)
     if not matched:
         return None
 
@@ -182,6 +184,7 @@ class LLMScore(Metric):
         category_key: A key to create category-wise mean score.
             The category key is expected to be in extra_info.
         metric_prefix: A prefix to be added to the metric keys in the summary and instance details.
+        regex_to_parse_score: A regular expression to parse score.
 
     Examples:
         >>> from flexeval import LLMScore, OpenAIChatAPI, Jinja2PromptTemplate
@@ -216,6 +219,7 @@ def __init__(
         valid_score_range: tuple[int, int] | None = None,
         category_key: str | None = None,
         metric_prefix: str | None = None,
+        regex_to_parse_score: str = r"(\d+)",
     ) -> None:
         self.language_model = language_model
         self.prompt_template = prompt_template
@@ -224,6 +228,7 @@ def __init__(
         self.valid_score_range = valid_score_range
         self.category_key = category_key
         self.metric_prefix = f"{metric_prefix}-" if metric_prefix else ""
+        self.regex_to_parse_score = regex_to_parse_score
 
     def evaluate(
         self,
@@ -254,6 +259,7 @@ def evaluate(
             evaluator_score = parse_score_from_evaluator_output(
                 evaluator_output.text,
                 valid_score_range=self.valid_score_range,
+                regex_to_parse_score=self.regex_to_parse_score,
             )
             if evaluator_score is None:
                 logger.warning(f"Failed to parse score from evaluator output: {evaluator_output}")

tests/core/metric/test_llm_score.py

@@ -34,20 +34,24 @@ def generate_chat_response(
 
 
 @pytest.mark.parametrize(
-    ("evaluator_output", "valid_score_range", "expected_score"),
+    ("evaluator_output", "valid_score_range", "regex_to_parse_score", "expected_score"),
     [
-        ("The final score is 65.", None, 65),
-        ("The final score is 65.", (0, 5), None),
-        ("Yes, this is a good one.", None, None),
-        ("The score is 5. No, it is 6.", None, 6),
+        ("The final score is 65.", None, r"(\d+)", 65),
+        ("The final score is 65.", (0, 5), r"(\d+)", None),
+        ("Yes, this is a good one.", None, r"(\d+)", None),
+        ("The score is 5. No, it is 6.", None, r"(\d+)", 6),
+        ("The score is [[5]]. No, it is 6.", None, r"\[\[(\d+)\]\]", 5),
     ],
 )
 def test_parse_score_from_evaluator_output(
     evaluator_output: str,
     valid_score_range: tuple[int, int] | None,
+    regex_to_parse_score: str,
     expected_score: int,
 ) -> None:
-    score = parse_score_from_evaluator_output(evaluator_output, valid_score_range)
+    score = parse_score_from_evaluator_output(
+        evaluator_output, valid_score_range, regex_to_parse_score=regex_to_parse_score
+    )
     assert score == expected_score