add judgeConfigKey

Author: edwinokonkwo

ldai/chat/__init__.py

@@ -112,7 +112,8 @@ async def evaluate_judge(judge_config):
             )
 
             if eval_result and eval_result.success:
-                self._tracker.track_eval_scores(eval_result.evals)
+                eval_result.judge_config_key = judge_config.key
+                self._tracker.track_judge_response(eval_result)
 
             return eval_result
 

ldai/providers/types.py

@@ -74,6 +74,7 @@ class JudgeResponse:
     """
     Response from a judge evaluation containing scores and reasoning for multiple metrics.
     """
+    judge_config_key: Optional[str] = None  # The key of the judge configuration that was used to generate this response
     evals: Dict[str, EvalScore]  # Dictionary where keys are metric names and values contain score and reasoning
     success: bool  # Whether the evaluation completed successfully
     error: Optional[str] = None  # Error message if evaluation failed
@@ -86,6 +87,8 @@ def to_dict(self) -> Dict[str, Any]:
             'evals': {key: eval_score.to_dict() for key, eval_score in self.evals.items()},
             'success': self.success,
         }
+        if self.judge_config_key is not None:
+            result['judgeConfigKey'] = self.judge_config_key
         if self.error is not None:
             result['error'] = self.error
         return result

ldai/tracker.py

@@ -224,22 +224,27 @@ def track_eval_scores(self, scores: Dict[str, Any]) -> None:
 
     def track_judge_response(self, judge_response: Any) -> None:
         """
-        Track a judge response, including evaluation scores and success status.
+        Track a judge response, including evaluation scores with judge config key.
 
         :param judge_response: JudgeResponse object containing evals and success status
         """
-        from ldai.providers.types import JudgeResponse
+        from ldai.providers.types import JudgeResponse, EvalScore
 
         if isinstance(judge_response, JudgeResponse):
-            # Track evaluation scores
+            # Track evaluation scores with judge config key included in metadata
             if judge_response.evals:
-                self.track_eval_scores(judge_response.evals)
-
-            # Track success/error based on judge response
-            if judge_response.success:
-                self.track_success()
-            else:
-                self.track_error()
+                track_data = self.__get_track_data()
+                if judge_response.judge_config_key:
+                    track_data = {**track_data, 'judgeConfigKey': judge_response.judge_config_key}
+                
+                for metric_key, eval_score in judge_response.evals.items():
+                    if isinstance(eval_score, EvalScore):
+                        self._ld_client.track(
+                            metric_key,
+                            self._context,
+                            track_data,
+                            eval_score.score
+                        )
 
     def track_feedback(self, feedback: Dict[str, FeedbackKind]) -> None:
         """