Telemetry

Author: pietrozullo

index.ts

@@ -13,4 +13,7 @@ export { BaseAdapter, LangChainAdapter } from './src/adapters/index.js'
 export { ServerManager } from './src/managers/server_manager.js'
 export * from './src/managers/tools/index.js'
 
+// Export telemetry utilities
+export { setTelemetrySource, Telemetry } from './src/telemetry/index.js'
+
 export { BaseConnector, HttpConnector, loadConfigFile, Logger, logger, MCPAgent, MCPClient, MCPSession, StdioConnector, WebSocketConnector }

package-lock.json

@@ -75,11 +75,13 @@
     "@langchain/core": "0.3.58",
     "@langchain/openai": "^0.5.15",
     "@modelcontextprotocol/sdk": "1.12.1",
+    "@scarf/scarf": "^1.4.0",
     "dotenv": "^16.5.0",
     "eventsource": "^3.0.6",
     "fastembed": "^1.14.4",
     "langchain": "^0.3.27",
     "lodash-es": "^4.17.21",
+    "posthog-node": "^5.1.1",
     "uuid": "^11.1.0",
     "winston": "^3.17.0",
     "ws": "^8.18.2",

package.json

@@ -24,6 +24,7 @@ import {
 import { LangChainAdapter } from '../adapters/langchain_adapter.js'
 import { logger } from '../logging.js'
 import { ServerManager } from '../managers/server_manager.js'
+import { extractModelInfo, Telemetry } from '../telemetry/index.js'
 import { createSystemMessage } from './prompts/system_prompt_builder.js'
 import { DEFAULT_SYSTEM_PROMPT_TEMPLATE, SERVER_MANAGER_SYSTEM_PROMPT_TEMPLATE } from './prompts/templates.js'
 
@@ -50,6 +51,9 @@ export class MCPAgent {
   private tools: StructuredToolInterface[] = []
   private adapter: LangChainAdapter
   private serverManager: ServerManager | null = null
+  private telemetry: Telemetry
+  private modelProvider: string
+  private modelName: string
 
   constructor(options: {
     llm: BaseLanguageModelInterface
@@ -98,6 +102,13 @@ export class MCPAgent {
     else {
       this.adapter = options.adapter ?? new LangChainAdapter(this.disallowedTools)
     }
+
+    // Initialize telemetry
+    this.telemetry = Telemetry.getInstance()
+    // Track model info for telemetry
+    const [provider, name] = extractModelInfo(this.llm as any)
+    this.modelProvider = provider
+    this.modelName = name
   }
 
   public async initialize(): Promise<void> {
@@ -298,6 +309,10 @@ export class MCPAgent {
   ): AsyncGenerator<AgentStep, string, void> {
     let result = ''
     let initializedHere = false
+    const startTime = Date.now()
+    const toolsUsedNames: string[] = []
+    let stepsTaken = 0
+    let success = false
 
     try {
       if (manageConnector && !this.initialized) {
@@ -340,6 +355,7 @@ export class MCPAgent {
       logger.info(`🏁 Starting agent execution with max_steps=${steps}`)
 
       for (let stepNum = 0; stepNum < steps; stepNum++) {
+        stepsTaken = stepNum + 1
         if (this.useServerManager && this.serverManager) {
           const currentTools = this.serverManager.tools
           const currentToolNames = new Set(currentTools.map(t => t.name))
@@ -385,6 +401,7 @@ export class MCPAgent {
             yield step
             const { action, observation } = step
             const toolName = action.tool
+            toolsUsedNames.push(toolName)
             let toolInputStr = String(action.toolInput)
             if (toolInputStr.length > 100)
               toolInputStr = `${toolInputStr.slice(0, 97)}...`
@@ -432,6 +449,7 @@ export class MCPAgent {
       }
 
       logger.info('🎉 Agent execution complete')
+      success = true
       return result
     }
     catch (e) {
@@ -443,6 +461,44 @@ export class MCPAgent {
       throw e
     }
     finally {
+      // Track comprehensive execution data
+      const executionTimeMs = Date.now() - startTime
+
+      let serverCount = 0
+      if (this.client) {
+        serverCount = Object.keys(await this.client.getAllActiveSessions()).length
+      }
+      else if (this.connectors) {
+        serverCount = this.connectors.length
+      }
+
+      const conversationHistoryLength = this.memoryEnabled ? this.conversationHistory.length : 0
+
+      await this.telemetry.trackAgentExecution({
+        executionMethod: 'stream',
+        query,
+        success,
+        modelProvider: this.modelProvider,
+        modelName: this.modelName,
+        serverCount,
+        serverIdentifiers: this.connectors.map(connector => connector.publicIdentifier),
+        totalToolsAvailable: this.tools.length,
+        toolsAvailableNames: this.tools.map(t => t.name),
+        maxStepsConfigured: this.maxSteps,
+        memoryEnabled: this.memoryEnabled,
+        useServerManager: this.useServerManager,
+        maxStepsUsed: maxSteps ?? null,
+        manageConnector,
+        externalHistoryUsed: externalHistory !== undefined,
+        stepsTaken,
+        toolsUsedCount: toolsUsedNames.length,
+        toolsUsedNames,
+        response: result,
+        executionTimeMs,
+        errorType: success ? null : 'execution_error',
+        conversationHistoryLength,
+      })
+
       if (manageConnector && !this.client && initializedHere) {
         logger.info('🧹 Closing agent after query completion')
         await this.close()

src/agents/mcp_agent.ts

@@ -33,6 +33,9 @@ export abstract class BaseConnector {
   /** Establish the connection and create the SDK client. */
   abstract connect(): Promise<void>
 
+  /** Get the identifier for the connector. */
+  abstract get publicIdentifier(): Record<string, string>
+
   /** Disconnect and release resources. */
   async disconnect(): Promise<void> {
     if (!this.connected) {

src/connectors/base.ts

@@ -70,4 +70,11 @@ export class HttpConnector extends BaseConnector {
       throw err
     }
   }
+
+  get publicIdentifier(): Record<string, string> {
+    return {
+      type: 'http',
+      url: this.baseUrl,
+    }
+  }
 }

src/connectors/http.ts

@@ -91,4 +91,11 @@ export class StdioConnector extends BaseConnector {
       throw err
     }
   }
+
+  get publicIdentifier(): Record<string, string> {
+    return {
+      'type': 'stdio',
+      'command&args': `${this.command} ${this.args.join(' ')}`,
+    }
+  }
 }

src/connectors/stdio.ts

@@ -183,4 +183,11 @@ export class WebSocketConnector extends BaseConnector {
 
     this.toolsCache = null
   }
+
+  get publicIdentifier(): Record<string, string> {
+    return {
+      type: 'websocket',
+      url: this.url,
+    }
+  }
 }

src/connectors/websocket.ts

@@ -0,0 +1,81 @@
+export abstract class BaseTelemetryEvent {
+  abstract get name(): string
+  abstract get properties(): Record<string, any>
+}
+
+export interface MCPAgentExecutionEventData {
+  // Execution method and context
+  executionMethod: string // "run" or "astream"
+  query: string // The actual user query
+  success: boolean
+
+  // Agent configuration
+  modelProvider: string
+  modelName: string
+  serverCount: number
+  serverIdentifiers: Array<Record<string, string>>
+  totalToolsAvailable: number
+  toolsAvailableNames: string[]
+  maxStepsConfigured: number
+  memoryEnabled: boolean
+  useServerManager: boolean
+
+  // Execution PARAMETERS
+  maxStepsUsed: number | null
+  manageConnector: boolean
+  externalHistoryUsed: boolean
+
+  // Execution results
+  stepsTaken?: number | null
+  toolsUsedCount?: number | null
+  toolsUsedNames?: string[] | null
+  response?: string | null // The actual response
+  executionTimeMs?: number | null
+  errorType?: string | null
+
+  // Context
+  conversationHistoryLength?: number | null
+}
+
+export class MCPAgentExecutionEvent extends BaseTelemetryEvent {
+  constructor(private data: MCPAgentExecutionEventData) {
+    super()
+  }
+
+  get name(): string {
+    return 'mcp_agent_execution'
+  }
+
+  get properties(): Record<string, any> {
+    return {
+      // Core execution info
+      execution_method: this.data.executionMethod,
+      query: this.data.query,
+      query_length: this.data.query.length,
+      success: this.data.success,
+      // Agent configuration
+      model_provider: this.data.modelProvider,
+      model_name: this.data.modelName,
+      server_count: this.data.serverCount,
+      server_identifiers: this.data.serverIdentifiers,
+      total_tools_available: this.data.totalToolsAvailable,
+      tools_available_names: this.data.toolsAvailableNames,
+      max_steps_configured: this.data.maxStepsConfigured,
+      memory_enabled: this.data.memoryEnabled,
+      use_server_manager: this.data.useServerManager,
+      // Execution parameters (always include, even if null)
+      max_steps_used: this.data.maxStepsUsed,
+      manage_connector: this.data.manageConnector,
+      external_history_used: this.data.externalHistoryUsed,
+      // Execution results (always include, even if null)
+      steps_taken: this.data.stepsTaken ?? null,
+      tools_used_count: this.data.toolsUsedCount ?? null,
+      tools_used_names: this.data.toolsUsedNames ?? null,
+      response: this.data.response ?? null,
+      response_length: this.data.response ? this.data.response.length : null,
+      execution_time_ms: this.data.executionTimeMs ?? null,
+      error_type: this.data.errorType ?? null,
+      conversation_history_length: this.data.conversationHistoryLength ?? null,
+    }
+  }
+}

src/telemetry/events.ts

@@ -0,0 +1,10 @@
+import { Telemetry } from './telemetry.js'
+
+export { BaseTelemetryEvent, MCPAgentExecutionEvent, MCPAgentExecutionEventData } from './events.js'
+export { Telemetry } from './telemetry.js'
+export { extractModelInfo, getModelName, getModelProvider, getPackageVersion } from './utils.js'
+
+// Convenience function to set telemetry source globally
+export function setTelemetrySource(source: string): void {
+  Telemetry.getInstance().setSource(source)
+}