fix: Include the AI Judge Config key with tracked metrics

Author: jsonbailey

packages/sdk/server-ai/src/LDAIClientImpl.ts

@@ -63,7 +63,7 @@ export class LDAIClientImpl implements LDAIClient {
       this._logger?.warn(
         `AI Config mode mismatch for ${key}: expected ${mode}, got ${flagMode}. Returning disabled config.`,
       );
-      return LDAIConfigUtils.createDisabledConfig(mode);
+      return LDAIConfigUtils.createDisabledConfig(key, mode);
     }
 
     const tracker = new LDAIConfigTrackerImpl(
@@ -78,7 +78,7 @@ export class LDAIClientImpl implements LDAIClient {
       context,
     );
 
-    const config = LDAIConfigUtils.fromFlagValue(value, tracker);
+    const config = LDAIConfigUtils.fromFlagValue(key, value, tracker);
 
     // Apply variable interpolation (always needed for ldctx)
     return this._applyInterpolation(config, context, variables);

packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts

@@ -78,7 +78,12 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker {
   trackEvalScores(scores: Record<string, EvalScore>) {
     // Track each evaluation score individually
     Object.entries(scores).forEach(([metricKey, evalScore]) => {
-      this._ldClient.track(metricKey, this._context, this.getTrackData(), evalScore.score);
+      this._ldClient.track(
+        metricKey,
+        this._context,
+        { ...this.getTrackData(), judgeConfigKey: evalScore.judgeConfigKey },
+        evalScore.score,
+      );
     });
   }
 

packages/sdk/server-ai/src/api/config/LDAIConfigUtils.ts

@@ -82,19 +82,23 @@ export class LDAIConfigUtils {
    * @param tracker The tracker to add to the config
    * @returns The appropriate AI configuration type
    */
-  static fromFlagValue(flagValue: LDAIConfigFlagValue, tracker: LDAIConfigTracker): LDAIConfigKind {
+  static fromFlagValue(
+    key: string,
+    flagValue: LDAIConfigFlagValue,
+    tracker: LDAIConfigTracker,
+  ): LDAIConfigKind {
     // Determine the actual mode from flag value
     // eslint-disable-next-line no-underscore-dangle
     const flagValueMode = flagValue._ldMeta?.mode;
 
     switch (flagValueMode) {
       case 'agent':
-        return this.toAgentConfig(flagValue, tracker);
+        return this.toAgentConfig(key, flagValue, tracker);
       case 'judge':
-        return this.toJudgeConfig(flagValue, tracker);
+        return this.toJudgeConfig(key, flagValue, tracker);
       case 'completion':
       default:
-        return this.toCompletionConfig(flagValue, tracker);
+        return this.toCompletionConfig(key, flagValue, tracker);
     }
   }
 
@@ -104,15 +108,17 @@ export class LDAIConfigUtils {
    * @param mode The mode for the disabled config
    * @returns A disabled config of the appropriate type
    */
-  static createDisabledConfig(mode: LDAIConfigMode): LDAIConfigKind {
+  static createDisabledConfig(key: string, mode: LDAIConfigMode): LDAIConfigKind {
     switch (mode) {
       case 'agent':
         return {
+          key,
           enabled: false,
           tracker: undefined,
         } as LDAIAgentConfig;
       case 'judge':
         return {
+          key,
           enabled: false,
           tracker: undefined,
           evaluationMetricKeys: [],
@@ -121,6 +127,7 @@ export class LDAIConfigUtils {
       default:
         // Default to completion config for completion mode or any unexpected mode
         return {
+          key,
           enabled: false,
           tracker: undefined,
         } as LDAICompletionConfig;
@@ -133,8 +140,9 @@ export class LDAIConfigUtils {
    * @param flagValue The flag value from LaunchDarkly
    * @returns Base configuration object
    */
-  private static _toBaseConfig(flagValue: LDAIConfigFlagValue) {
+  private static _toBaseConfig(key: string, flagValue: LDAIConfigFlagValue) {
     return {
+      key,
       // eslint-disable-next-line no-underscore-dangle
       enabled: flagValue._ldMeta?.enabled ?? false,
       model: flagValue.model,
@@ -150,11 +158,12 @@ export class LDAIConfigUtils {
    * @returns A completion configuration
    */
   static toCompletionConfig(
+    key: string,
     flagValue: LDAIConfigFlagValue,
     tracker: LDAIConfigTracker,
   ): LDAICompletionConfig {
     return {
-      ...this._toBaseConfig(flagValue),
+      ...this._toBaseConfig(key, flagValue),
       tracker,
       messages: flagValue.messages,
       judgeConfiguration: flagValue.judgeConfiguration,
@@ -169,11 +178,12 @@ export class LDAIConfigUtils {
    * @returns An agent configuration
    */
   static toAgentConfig(
+    key: string,
     flagValue: LDAIConfigFlagValue,
     tracker: LDAIConfigTracker,
   ): LDAIAgentConfig {
     return {
-      ...this._toBaseConfig(flagValue),
+      ...this._toBaseConfig(key, flagValue),
       tracker,
       instructions: flagValue.instructions,
       judgeConfiguration: flagValue.judgeConfiguration,
@@ -188,11 +198,12 @@ export class LDAIConfigUtils {
    * @returns A judge configuration
    */
   static toJudgeConfig(
+    key: string,
     flagValue: LDAIConfigFlagValue,
     tracker: LDAIConfigTracker,
   ): LDAIJudgeConfig {
     return {
-      ...this._toBaseConfig(flagValue),
+      ...this._toBaseConfig(key, flagValue),
       tracker,
       messages: flagValue.messages,
       evaluationMetricKeys: flagValue.evaluationMetricKeys || [],

packages/sdk/server-ai/src/api/config/types.ts

@@ -95,6 +95,10 @@ export interface LDAIConfigDefault {
  * Base AI Config interface without mode-specific fields.
  */
 export interface LDAIConfig extends Omit<LDAIConfigDefault, 'enabled'> {
+  /**
+   * The key of the AI Config.
+   */
+  key: string;
   /**
    * Whether the configuration is enabled.
    */

packages/sdk/server-ai/src/api/judge/Judge.ts

@@ -208,6 +208,7 @@ export class Judge {
       results[metricKey] = {
         score: evalData.score,
         reasoning: evalData.reasoning,
+        judgeConfigKey: this._aiConfig.key,
       };
     });
 

packages/sdk/server-ai/src/api/judge/types.ts

@@ -24,6 +24,8 @@ export interface EvalScore {
   score: number;
   /** Reasoning behind the provided score for this metric */
   reasoning: string;
+  /** The key of the judge configuration that was used to evaluate this metric */
+  judgeConfigKey?: string;
 }
 
 /**