[agent builder] telemetry: tokens, ttl and tool cals by model (elastic#246688)

Author: ppisljar

x-pack/platform/plugins/private/telemetry_collection_xpack/schema/xpack_platform.json

@@ -628,6 +628,93 @@
             }
           }
         },
+        "tokens_by_model": {
+          "type": "array",
+          "items": {
+            "properties": {
+              "model": {
+                "type": "keyword",
+                "_meta": {
+                  "description": "Model identifier for token usage grouping"
+                }
+              },
+              "total_tokens": {
+                "type": "long",
+                "_meta": {
+                  "description": "Total tokens (input + output) consumed by this model"
+                }
+              },
+              "avg_tokens_per_round": {
+                "type": "float",
+                "_meta": {
+                  "description": "Average tokens per conversation round for this model"
+                }
+              },
+              "sample_count": {
+                "type": "long",
+                "_meta": {
+                  "description": "Number of rounds sampled for this model"
+                }
+              }
+            }
+          }
+        },
+        "query_to_result_time_by_model": {
+          "type": "array",
+          "items": {
+            "properties": {
+              "model": {
+                "type": "keyword",
+                "_meta": {
+                  "description": "Model identifier for QTRT grouping"
+                }
+              },
+              "p50": {
+                "type": "long"
+              },
+              "p75": {
+                "type": "long"
+              },
+              "p90": {
+                "type": "long"
+              },
+              "p95": {
+                "type": "long"
+              },
+              "p99": {
+                "type": "long"
+              },
+              "mean": {
+                "type": "long"
+              },
+              "total_samples": {
+                "type": "long"
+              },
+              "sample_count": {
+                "type": "long"
+              }
+            }
+          }
+        },
+        "tool_calls_by_model": {
+          "type": "array",
+          "items": {
+            "properties": {
+              "model": {
+                "type": "keyword",
+                "_meta": {
+                  "description": "Model identifier for tool-call grouping"
+                }
+              },
+              "count": {
+                "type": "long",
+                "_meta": {
+                  "description": "Tool calls counted for this model"
+                }
+              }
+            }
+          }
+        },
         "tool_calls": {
           "properties": {
             "total": {

x-pack/platform/plugins/shared/agent_builder/server/telemetry/query_utils.ts

@@ -647,6 +647,265 @@ export class QueryUtils {
     }
   }
 
+  /**
+   * Get token consumption grouped by model
+   */
+  async getTokensByModel(): Promise<
+    Array<{
+      model: string;
+      total_tokens: number;
+      avg_tokens_per_round: number;
+      sample_count: number;
+    }>
+  > {
+    try {
+      const conversationIndexName = chatSystemIndex('conversations');
+      const response = await this.esClient.search({
+        index: conversationIndexName,
+        size: 0,
+        aggs: {
+          all_rounds: {
+            nested: {
+              path: 'conversation_rounds',
+            },
+            aggs: {
+              by_model: {
+                terms: {
+                  field: 'conversation_rounds.model_usage.model',
+                  size: 50,
+                  missing: 'unknown',
+                },
+                aggs: {
+                  input_tokens: {
+                    sum: {
+                      field: 'conversation_rounds.model_usage.input_tokens',
+                    },
+                  },
+                  output_tokens: {
+                    sum: {
+                      field: 'conversation_rounds.model_usage.output_tokens',
+                    },
+                  },
+                },
+              },
+            },
+          },
+        },
+      });
+
+      const buckets = (response.aggregations?.all_rounds as any)?.by_model?.buckets || [];
+
+      const results: Array<{
+        model: string;
+        total_tokens: number;
+        avg_tokens_per_round: number;
+        sample_count: number;
+      }> = [];
+
+      for (const bucket of buckets) {
+        const inputTokens = bucket.input_tokens?.value || 0;
+        const outputTokens = bucket.output_tokens?.value || 0;
+        const totalTokens = inputTokens + outputTokens;
+        const sampleCount = bucket.doc_count || 0;
+        const avgTokensPerRound =
+          sampleCount > 0 ? Math.round((totalTokens / sampleCount) * 100) / 100 : 0;
+
+        results.push({
+          model: bucket.key as string,
+          total_tokens: Math.round(totalTokens),
+          avg_tokens_per_round: avgTokensPerRound,
+          sample_count: sampleCount,
+        });
+      }
+
+      results.sort((a, b) => b.total_tokens - a.total_tokens);
+
+      return results;
+    } catch (error) {
+      if (!isIndexNotFoundError(error)) {
+        this.logger.warn(`Failed to fetch tokens by model: ${error.message}`);
+      }
+      return [];
+    }
+  }
+
+  /**
+   * Get query-to-result time (TTLT) grouped by model
+   */
+  async getQueryToResultTimeByModel(): Promise<
+    Array<{
+      model: string;
+      p50: number;
+      p75: number;
+      p90: number;
+      p95: number;
+      p99: number;
+      mean: number;
+      total_samples: number;
+      sample_count: number;
+    }>
+  > {
+    try {
+      const conversationIndexName = chatSystemIndex('conversations');
+      const response = await this.esClient.search({
+        index: conversationIndexName,
+        size: 0,
+        aggs: {
+          all_rounds: {
+            nested: {
+              path: 'conversation_rounds',
+            },
+            aggs: {
+              by_model: {
+                terms: {
+                  field: 'conversation_rounds.model_usage.model',
+                  size: 50,
+                  missing: 'unknown',
+                },
+                aggs: {
+                  ttl_percentiles: {
+                    percentiles: {
+                      field: 'conversation_rounds.time_to_last_token',
+                      percents: [50, 75, 90, 95, 99],
+                    },
+                  },
+                  ttl_avg: {
+                    avg: {
+                      field: 'conversation_rounds.time_to_last_token',
+                    },
+                  },
+                  ttl_count: {
+                    value_count: {
+                      field: 'conversation_rounds.time_to_last_token',
+                    },
+                  },
+                },
+              },
+            },
+          },
+        },
+      });
+
+      const buckets = (response.aggregations?.all_rounds as any)?.by_model?.buckets || [];
+
+      const results: Array<{
+        model: string;
+        p50: number;
+        p75: number;
+        p90: number;
+        p95: number;
+        p99: number;
+        mean: number;
+        total_samples: number;
+        sample_count: number;
+      }> = [];
+
+      for (const bucket of buckets) {
+        const percentiles = bucket.ttl_percentiles?.values || {};
+        results.push({
+          model: bucket.key as string,
+          p50: Math.round(percentiles['50.0'] || 0),
+          p75: Math.round(percentiles['75.0'] || 0),
+          p90: Math.round(percentiles['90.0'] || 0),
+          p95: Math.round(percentiles['95.0'] || 0),
+          p99: Math.round(percentiles['99.0'] || 0),
+          mean: Math.round(bucket.ttl_avg?.value || 0),
+          total_samples: bucket.ttl_count?.value || 0,
+          sample_count: bucket.doc_count || 0,
+        });
+      }
+
+      results.sort((a, b) => b.sample_count - a.sample_count);
+
+      return results;
+    } catch (error) {
+      if (!isIndexNotFoundError(error)) {
+        this.logger.warn(`Failed to fetch query-to-result time by model: ${error.message}`);
+      }
+      return [];
+    }
+  }
+
+  /**
+   * Get tool call counts grouped by model based on round steps
+   */
+  async getToolCallsByModel(): Promise<
+    Array<{
+      model: string;
+      count: number;
+    }>
+  > {
+    try {
+      const conversationIndexName = chatSystemIndex('conversations');
+      const response = await this.esClient.search({
+        index: conversationIndexName,
+        size: 0,
+        aggs: {
+          tool_calls_by_model: {
+            scripted_metric: {
+              init_script: 'state.modelCalls = new HashMap();',
+              map_script: `
+                def rounds = params._source.conversation_rounds;
+                if (rounds == null) return;
+                for (def round : rounds) {
+                  def modelUsage = round.model_usage;
+                  def model = (modelUsage != null && modelUsage.model != null) ? modelUsage.model : 'unknown';
+                  def steps = round.steps;
+                  if (steps == null) continue;
+                  int callCount = 0;
+                  for (def step : steps) {
+                    if (step.type != null && step.type == 'tool_call') {
+                      callCount += 1;
+                    }
+                  }
+                  if (callCount == 0) continue;
+                  def current = state.modelCalls.get(model);
+                  if (current == null) {
+                    state.modelCalls.put(model, callCount);
+                  } else {
+                    state.modelCalls.put(model, current + callCount);
+                  }
+                }
+              `,
+              combine_script: 'return state.modelCalls;',
+              reduce_script: `
+                Map combined = new HashMap();
+                for (state in states) {
+                  for (entry in state.entrySet()) {
+                    def model = entry.getKey();
+                    def value = entry.getValue();
+                    if (combined.containsKey(model)) {
+                      combined.put(model, combined.get(model) + value);
+                    } else {
+                      combined.put(model, value);
+                    }
+                  }
+                }
+                return combined;
+              `,
+            },
+          },
+        },
+      });
+
+      const aggregated = (response.aggregations as any)?.tool_calls_by_model?.value || {};
+      const results: Array<{ model: string; count: number }> = [];
+
+      for (const [model, count] of Object.entries(aggregated)) {
+        results.push({ model, count: Number(count) });
+      }
+
+      results.sort((a, b) => b.count - a.count);
+
+      return results;
+    } catch (error) {
+      if (!isIndexNotFoundError(error)) {
+        this.logger.warn(`Failed to fetch tool calls by model: ${error.message}`);
+      }
+      return [];
+    }
+  }
+
   /**
    * Calculate percentiles from bucketed time data
    * @param buckets - Map of bucket name → count