[workchat] add tracing configuration (#217825)

## Summary

Allow enabling langsmith tracing via kibana config file

### Example

```yaml
xpack.workchatApp.tracing.langsmith:
  enabled: true
  apiKey: {API-KEY}
  project: {project-name}
```

Author: pgayvallet

x-pack/solutions/chat/plugins/workchat-app/README.md

@@ -1,3 +1,17 @@
 # WorkChatApp
 
 WorkChat application plugin
+
+
+## Enabling tracing
+
+### Langsmith tracing
+
+Enabling langsmith tracing can be done by adding the corresponding entry to your Kibana dev config file:
+
+```yaml
+xpack.workchatApp.tracing.langsmith:
+  enabled: true
+  apiKey: {API-KEY}
+  project: {project-name}
+```

x-pack/solutions/chat/plugins/workchat-app/server/config.ts

@@ -10,10 +10,25 @@ import { PluginConfigDescriptor } from '@kbn/core/server';
 
 const configSchema = schema.object({
   enabled: schema.boolean({ defaultValue: false }),
+  tracing: schema.object(
+    {
+      langsmith: schema.maybe(
+        schema.object({
+          enabled: schema.boolean({ defaultValue: false }),
+          apiKey: schema.string(),
+          apiUrl: schema.string({ defaultValue: 'https://api.smith.langchain.com' }),
+          project: schema.string(),
+        })
+      ),
+    },
+    { defaultValue: {} }
+  ),
 });
 
 export type WorkChatAppConfig = TypeOf<typeof configSchema>;
 
+export type WorkChatTracingConfig = WorkChatAppConfig['tracing'];
+
 export const config: PluginConfigDescriptor<WorkChatAppConfig> = {
   exposeToBrowser: {},
   schema: configSchema,

x-pack/solutions/chat/plugins/workchat-app/server/plugin.ts

@@ -18,6 +18,7 @@ import { registerFeatures } from './features';
 import type { InternalServices } from './services/types';
 import { IntegrationRegistry } from './services/integrations';
 import { createServices } from './services/create_services';
+import type { WorkChatAppConfig } from './config';
 import type {
   WorkChatAppPluginSetup,
   WorkChatAppPluginStart,
@@ -35,11 +36,13 @@ export class WorkChatAppPlugin
     >
 {
   private readonly logger: LoggerFactory;
+  private readonly config: WorkChatAppConfig;
   private readonly integrationRegistry = new IntegrationRegistry();
   private services?: InternalServices;
 
   constructor(context: PluginInitializerContext) {
     this.logger = context.logger;
+    this.config = context.config.get<WorkChatAppConfig>();
   }
 
   public setup(
@@ -78,6 +81,7 @@ export class WorkChatAppPlugin
   ): WorkChatAppPluginStart {
     this.services = createServices({
       core,
+      config: this.config,
       logger: this.logger,
       pluginsDependencies,
       integrationRegistry: this.integrationRegistry,

x-pack/solutions/chat/plugins/workchat-app/server/services/create_services.ts

@@ -8,6 +8,7 @@
 import type { CoreStart } from '@kbn/core/server';
 import type { LoggerFactory } from '@kbn/core/server';
 import type { WorkChatAppPluginStartDependencies } from '../types';
+import type { WorkChatAppConfig } from '../config';
 import type { InternalServices } from './types';
 import { IntegrationsServiceImpl } from './integrations/integrations_service';
 import { ConversationServiceImpl } from './conversations';
@@ -18,13 +19,15 @@ import { IntegrationRegistry } from './integrations';
 
 interface CreateServicesParams {
   core: CoreStart;
+  config: WorkChatAppConfig;
   logger: LoggerFactory;
   pluginsDependencies: WorkChatAppPluginStartDependencies;
   integrationRegistry: IntegrationRegistry;
 }
 
 export function createServices({
   core,
+  config,
   logger,
   pluginsDependencies,
   integrationRegistry,
@@ -53,6 +56,7 @@ export function createServices({
 
   const agentFactory = new AgentFactory({
     inference: pluginsDependencies.inference,
+    tracingConfig: config.tracing,
     logger: logger.get('services.agentFactory'),
     agentService,
     integrationsService,

x-pack/solutions/chat/plugins/workchat-app/server/services/orchestration/agent_factory.ts

@@ -7,6 +7,7 @@
 
 import { KibanaRequest, Logger } from '@kbn/core/server';
 import { InferenceServerStart } from '@kbn/inference-plugin/server';
+import type { WorkChatTracingConfig } from '../../config';
 import { IntegrationsServiceImpl } from '../integrations/integrations_service';
 import type { AgentService } from '../agents';
 import type { AgentRunner } from './types';
@@ -19,19 +20,28 @@ interface AgentFactoryArgs {
   inference: InferenceServerStart;
   agentService: AgentService;
   integrationsService: IntegrationsServiceImpl;
+  tracingConfig: WorkChatTracingConfig;
 }
 
 export class AgentFactory {
   private readonly inference: InferenceServerStart;
   private readonly logger: Logger;
   private readonly agentService: AgentService;
   private readonly integrationsService: IntegrationsServiceImpl;
+  private readonly tracingConfig: WorkChatTracingConfig;
 
-  constructor({ inference, logger, integrationsService, agentService }: AgentFactoryArgs) {
+  constructor({
+    inference,
+    logger,
+    integrationsService,
+    agentService,
+    tracingConfig,
+  }: AgentFactoryArgs) {
     this.inference = inference;
     this.logger = logger;
     this.integrationsService = integrationsService;
     this.agentService = agentService;
+    this.tracingConfig = tracingConfig;
   }
 
   async getAgentRunner({
@@ -61,7 +71,13 @@ export class AgentFactory {
       chatModelOptions: {},
     });
 
-    return await createAgentRunner({ agent, chatModel, createSession, logger: this.logger });
+    return await createAgentRunner({
+      agent,
+      chatModel,
+      createSession,
+      logger: this.logger,
+      tracingConfig: this.tracingConfig,
+    });
   }
 
   private async createGatewaySession({

x-pack/solutions/chat/plugins/workchat-app/server/services/orchestration/agent_runner.ts

@@ -10,26 +10,31 @@ import { StreamEvent } from '@langchain/core/tracers/log_stream';
 import type { Logger } from '@kbn/core/server';
 import type { InferenceChatModel } from '@kbn/inference-langchain';
 import type { Agent } from '../../../common/agents';
+import type { WorkChatTracingConfig } from '../../config';
 import { createAgentGraph } from './agent_graph';
 import { conversationEventsToMessages } from './utils';
 import { convertGraphEvents } from './graph_events';
 import type { AgentRunner, AgentRunResult } from './types';
 import type { McpGatewaySession } from './mcp_gateway';
 import { graphNames } from './constants';
 import { getGraphMeta } from './graph_events';
+import { getTracers } from './tracing';
 
 export const createAgentRunner = async ({
   logger,
   agent,
   chatModel,
   createSession,
+  tracingConfig,
 }: {
   logger: Logger;
   agent: Agent;
   chatModel: InferenceChatModel;
   createSession: () => Promise<McpGatewaySession>;
+  tracingConfig: WorkChatTracingConfig;
 }): Promise<AgentRunner> => {
   const session = await createSession();
+  const tracers = getTracers({ config: tracingConfig });
 
   const closeSession = () => {
     session.close().catch((err) => {
@@ -55,6 +60,7 @@ export const createAgentRunner = async ({
             agentId: agent.id,
           },
           recursionLimit: 10,
+          callbacks: [...tracers],
         }
       );
 

x-pack/solutions/chat/plugins/workchat-app/server/services/orchestration/tracing/get_tracers.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { LangChainTracer } from '@langchain/core/tracers/tracer_langchain';
+import type { WorkChatTracingConfig } from '../../../config';
+import { getLangsmithTracer } from './langsmith';
+
+export const getTracers = ({ config }: { config: WorkChatTracingConfig }) => {
+  const tracers: LangChainTracer[] = [];
+  if (config?.langsmith?.enabled) {
+    tracers.push(
+      getLangsmithTracer({
+        apiKey: config.langsmith.apiKey,
+        apiUrl: config.langsmith.apiUrl,
+        project: config.langsmith.project,
+      })
+    );
+  }
+  return tracers;
+};

x-pack/solutions/chat/plugins/workchat-app/server/services/orchestration/tracing/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { getTracers } from './get_tracers';

x-pack/solutions/chat/plugins/workchat-app/server/services/orchestration/tracing/langsmith.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Client } from 'langsmith';
+import { LangChainTracer } from '@langchain/core/tracers/tracer_langchain';
+
+export interface LangsmithTracingConfig {
+  apiKey: string;
+  apiUrl: string;
+  project: string;
+}
+
+export const getLangsmithTracer = (config: LangsmithTracingConfig): LangChainTracer => {
+  return new LangChainTracer({
+    projectName: config.project,
+    client: new Client({ apiKey: config.apiKey, apiUrl: config.apiUrl }),
+  });
+};