move key to judge response

Author: jsonbailey

packages/sdk/server-ai/__tests__/Judge.test.ts

@@ -110,20 +110,18 @@ describe('Judge', () => {
           relevance: {
             score: 0.8,
             reasoning: 'The response is relevant to the question',
-            judgeConfigKey: 'test-judge',
           },
           accuracy: {
             score: 0.9,
             reasoning: 'The response is factually accurate',
-            judgeConfigKey: 'test-judge',
           },
           helpfulness: {
             score: 0.7,
             reasoning: 'The response provides helpful information',
-            judgeConfigKey: 'test-judge',
           },
         },
         success: true,
+        judgeConfigKey: 'test-judge',
       });
 
       expect(mockProvider.invokeStructuredModel).toHaveBeenCalledWith(
@@ -267,10 +265,11 @@ describe('Judge', () => {
       // When one metric is missing, it returns the partial evals it has with success: false
       expect(result).toEqual({
         evals: {
-          relevance: { score: 0.8, reasoning: 'Good', judgeConfigKey: 'test-judge' },
-          helpfulness: { score: 0.7, reasoning: 'Helpful', judgeConfigKey: 'test-judge' },
+          relevance: { score: 0.8, reasoning: 'Good' },
+          helpfulness: { score: 0.7, reasoning: 'Helpful' },
         },
         success: false,
+        judgeConfigKey: 'test-judge',
       });
     });
 
@@ -302,6 +301,7 @@ describe('Judge', () => {
       expect(result).toEqual({
         evals: {},
         success: false,
+        judgeConfigKey: 'test-judge',
       });
     });
 
@@ -315,6 +315,7 @@ describe('Judge', () => {
         evals: {},
         success: false,
         error: 'Provider error',
+        judgeConfigKey: 'test-judge',
       });
       expect(mockLogger.error).toHaveBeenCalledWith('Judge evaluation failed:', error);
     });
@@ -328,6 +329,7 @@ describe('Judge', () => {
         evals: {},
         success: false,
         error: 'Unknown error',
+        judgeConfigKey: 'test-judge',
       });
     });
   });
@@ -380,20 +382,18 @@ describe('Judge', () => {
           relevance: {
             score: 0.8,
             reasoning: 'The response is relevant to the question',
-            judgeConfigKey: 'test-judge',
           },
           accuracy: {
             score: 0.9,
             reasoning: 'The response is factually accurate',
-            judgeConfigKey: 'test-judge',
           },
           helpfulness: {
             score: 0.7,
             reasoning: 'The response provides helpful information',
-            judgeConfigKey: 'test-judge',
           },
         },
         success: true,
+        judgeConfigKey: 'test-judge',
       });
 
       expect(mockProvider.invokeStructuredModel).toHaveBeenCalledWith(
@@ -479,9 +479,9 @@ describe('Judge', () => {
       const result = parseResponse(responseData);
 
       expect(result).toEqual({
-        relevance: { score: 0.8, reasoning: 'Good', judgeConfigKey: 'test-judge' },
-        accuracy: { score: 0.9, reasoning: 'Accurate', judgeConfigKey: 'test-judge' },
-        helpfulness: { score: 0.7, reasoning: 'Helpful', judgeConfigKey: 'test-judge' },
+        relevance: { score: 0.8, reasoning: 'Good' },
+        accuracy: { score: 0.9, reasoning: 'Accurate' },
+        helpfulness: { score: 0.7, reasoning: 'Helpful' },
       });
     });
 
@@ -514,7 +514,7 @@ describe('Judge', () => {
 
       // Only helpfulness passes validation, relevance and accuracy are skipped
       expect(result).toEqual({
-        helpfulness: { score: 0.7, reasoning: 'Helpful', judgeConfigKey: 'test-judge' },
+        helpfulness: { score: 0.7, reasoning: 'Helpful' },
       });
     });
   });

packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts

@@ -3,7 +3,7 @@ import { LDContext } from '@launchdarkly/js-server-sdk-common';
 import { name as aiSdkName, version as aiSdkVersion } from '../package.json';
 import { LDAIConfigTracker } from './api/config';
 import { LDAIMetricSummary } from './api/config/LDAIConfigTracker';
-import { EvalScore } from './api/judge/types';
+import { EvalScore, JudgeResponse } from './api/judge/types';
 import {
   createBedrockTokenUsage,
   createOpenAiUsage,
@@ -76,12 +76,17 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker {
   }
 
   trackEvalScores(scores: Record<string, EvalScore>) {
-    // Track each evaluation score individually
     Object.entries(scores).forEach(([metricKey, evalScore]) => {
+      this._ldClient.track(metricKey, this._context, this.getTrackData(), evalScore.score);
+    });
+  }
+
+  trackJudgeResponse(response: JudgeResponse) {
+    Object.entries(response.evals).forEach(([metricKey, evalScore]) => {
       this._ldClient.track(
         metricKey,
         this._context,
-        { ...this.getTrackData(), judgeConfigKey: evalScore.judgeConfigKey },
+        { ...this.getTrackData(), judgeConfigKey: response.judgeConfigKey },
         evalScore.score,
       );
     });

packages/sdk/server-ai/src/api/chat/TrackedChat.ts

@@ -84,13 +84,17 @@ export class TrackedChat {
         return undefined;
       }
 
-      const evalResult = await judge.evaluateMessages(messages, response, judgeConfig.samplingRate);
-
-      if (evalResult && evalResult.success) {
-        this.tracker.trackEvalScores(evalResult.evals);
+      const judgeResponse = await judge.evaluateMessages(
+        messages,
+        response,
+        judgeConfig.samplingRate,
+      );
+
+      if (judgeResponse && judgeResponse.success) {
+        this.tracker.trackJudgeResponse(judgeResponse);
       }
 
-      return evalResult;
+      return judgeResponse;
     });
 
     // ensure all evaluations complete even if some fail

packages/sdk/server-ai/src/api/config/LDAIConfigTracker.ts

@@ -1,4 +1,4 @@
-import { EvalScore } from '../judge/types';
+import { EvalScore, JudgeResponse } from '../judge/types';
 import { LDAIMetrics, LDFeedbackKind, LDTokenUsage } from '../metrics';
 
 /**
@@ -94,6 +94,13 @@ export interface LDAIConfigTracker {
    */
   trackEvalScores(scores: Record<string, EvalScore>): void;
 
+  /**
+   * Track a judge response containing evaluation scores and judge configuration key.
+   *
+   * @param response Judge response containing evaluation scores and judge configuration key
+   */
+  trackJudgeResponse(response: JudgeResponse): void;
+
   /**
    * Track the duration of execution of the provided function.
    *

packages/sdk/server-ai/src/api/judge/Judge.ts

@@ -91,13 +91,15 @@ export class Judge {
       return {
         evals,
         success,
+        judgeConfigKey: this._aiConfig.key,
       };
     } catch (error) {
       this._logger?.error('Judge evaluation failed:', error);
       return {
         evals: {},
         success: false,
         error: error instanceof Error ? error.message : 'Unknown error',
+        judgeConfigKey: this._aiConfig.key,
       };
     }
   }
@@ -208,7 +210,6 @@ export class Judge {
       results[metricKey] = {
         score: evalData.score,
         reasoning: evalData.reasoning,
-        judgeConfigKey: this._aiConfig.key,
       };
     });
 

packages/sdk/server-ai/src/api/judge/types.ts

@@ -24,14 +24,14 @@ export interface EvalScore {
   score: number;
   /** Reasoning behind the provided score for this metric */
   reasoning: string;
-  /** The key of the judge configuration that was used to evaluate this metric */
-  judgeConfigKey?: string;
 }
 
 /**
  * Response from a judge evaluation containing scores and reasoning for multiple metrics.
  */
 export interface JudgeResponse {
+  /** The key of the judge configuration that was used to generate this response */
+  judgeConfigKey?: string;
   /** Dictionary where keys are metric names and values contain score and reasoning */
   evals: Record<string, EvalScore>;
   /** Whether the evaluation completed successfully */