Add ProbabilisticCustomMetric

Author: pantonante

continuous_eval/metrics/custom/__init__.py

@@ -1 +1 @@
-from .custom_metric import CustomMetric, Example
+from .custom_metric import ProbabilisticCustomMetric,CustomMetric, Example

continuous_eval/metrics/custom/custom_metric.py

@@ -5,10 +5,14 @@
 from jinja2 import BaseLoader, Environment
 
 from continuous_eval.llms import LLMFactory
+from continuous_eval.metrics.base import (
+    Arg,
+    Field,
+    MetricPrompt,
+    response_type,
+)
 from continuous_eval.metrics.base.llm import LLMMetric
-from continuous_eval.metrics.base.metric import Arg, Field
-from continuous_eval.metrics.base.prompt import MetricPrompt
-from continuous_eval.metrics.base.response_type import JSON
+from continuous_eval.metrics.base.probabilistic import ProbabilisticMetric
 
 _CWD = Path(__file__).parent
 
@@ -49,7 +53,7 @@ def __init__(
         self.prompt = MetricPrompt(
             sys_prompt,
             user_prompt,
-            response_format=JSON(
+            response_format=response_type.JSON(
                 {k: v.type for k, v in response_format.items()}
             ),
         )
@@ -60,3 +64,52 @@ def __init__(
     @property
     def help(self):
         return self._criteria
+
+
+class ProbabilisticCustomMetric(ProbabilisticMetric):
+    def __init__(
+        self,
+        name: str,
+        criteria: str,
+        rubric: str,
+        arguments: Dict[str, Arg],
+        response_format: response_type.ResponseFormatBaseType,
+        examples: Optional[List[Example]] = None,
+        temperature: float = 1.0,
+        model: str = LLMFactory.default(),
+    ):
+        if not isinstance(
+            response_format, response_type.ResponseFormatBaseType
+        ):
+            raise ValueError("response_format must be a ResponseFormatBaseType")
+        if isinstance(response_format, response_type.JSON):
+            raise ValueError(
+                "Probabilistic metrics do not support JSON response format, use CustomMetric instead"
+            )
+        with open(_CWD / "custom_metric_sys_probabilistic.jinja2") as f:
+            raw_system_prompt = f.read()
+        with open(_CWD / "custom_metric_user.jinja2") as f:
+            raw_user_prompt = f.read()
+        env = Environment(loader=BaseLoader())
+        sys_prompt_template = env.from_string(raw_system_prompt)
+        user_prompt_template = env.from_string(raw_user_prompt)
+        sys_prompt = sys_prompt_template.render(
+            criteria=criteria,
+            rubric=rubric,
+            examples=examples,
+            response_format=response_format,
+        )
+        user_prompt = user_prompt_template.render(arguments=arguments)
+        self._criteria = criteria
+        self.prompt = MetricPrompt(
+            sys_prompt,
+            user_prompt,
+            response_format=response_format,
+        )
+        super().__init__(
+            name=name, prompt=self.prompt, temperature=temperature, model=model
+        )
+
+    @property
+    def help(self):
+        return self._criteria

continuous_eval/metrics/custom/custom_metric_sys_probabilistic.jinja2

@@ -0,0 +1,22 @@
+You are an expert evaluator system for a somewhat intelligent system.
+You need to evaluate the following criteria:
+{{ criteria }}
+
+-- GUIDELINES --
+When evaluating the answer, strictly adhere to the following guidelines:
+{{ rubric }}
+-- END OF GUIDELINES --
+
+{% if examples %}
+-- EXAMPLES --
+{%- for example in examples %}
+INPUT:
+{%- for key, value in example.input.items() %}
+{{ key }}: `{{ value }}`
+{%- endfor %}
+
+EVALUATION:
+{{ example.output | tojson }}
+--
+{%- endfor %} END OF EXAMPLES --
+{% endif %}

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "continuous-eval"
-version = "0.3.14post1"
+version = "0.3.14post2"
 description = "Open-Source Evaluation for GenAI Applications."
 authors = ["Yi Zhang <[email protected]>", "Pasquale Antonante <[email protected]>"]
 readme = "README.md"