return success false if we do not support the result

This is debatable since the model did produce a result. Returning not success allows users to rollback an invalid config that generates an unsupported response so we are taking a more cautious approach.

Author: jsonbailey

packages/ai-providers/server-ai-langchain/__tests__/LangChainProvider.test.ts

@@ -7,6 +7,14 @@ jest.mock('langchain/chat_models/universal', () => ({
   initChatModel: jest.fn(),
 }));
 
+// Mock logger
+const mockLogger = {
+  warn: jest.fn(),
+  info: jest.fn(),
+  error: jest.fn(),
+  debug: jest.fn(),
+};
+
 describe('LangChainProvider', () => {
   describe('convertMessagesToLangChain', () => {
     it('converts system messages to SystemMessage', () => {
@@ -100,6 +108,61 @@ describe('LangChainProvider', () => {
     });
   });
 
+  describe('invokeModel', () => {
+    let mockLLM: any;
+    let provider: LangChainProvider;
+
+    beforeEach(() => {
+      mockLLM = {
+        invoke: jest.fn(),
+      };
+      provider = new LangChainProvider(mockLLM, mockLogger);
+      jest.clearAllMocks();
+    });
+
+    it('returns success=true for string content', async () => {
+      const mockResponse = new AIMessage('Test response');
+      mockLLM.invoke.mockResolvedValue(mockResponse);
+
+      const messages = [{ role: 'user' as const, content: 'Hello' }];
+      const result = await provider.invokeModel(messages);
+
+      expect(result.metrics.success).toBe(true);
+      expect(result.message.content).toBe('Test response');
+      expect(mockLogger.warn).not.toHaveBeenCalled();
+    });
+
+    it('returns success=false for non-string content and logs warning', async () => {
+      const mockResponse = new AIMessage({ type: 'image', data: 'base64data' } as any);
+      mockLLM.invoke.mockResolvedValue(mockResponse);
+
+      const messages = [{ role: 'user' as const, content: 'Hello' }];
+      const result = await provider.invokeModel(messages);
+
+      expect(result.metrics.success).toBe(false);
+      expect(result.message.content).toBe('');
+      expect(mockLogger.warn).toHaveBeenCalledWith(
+        'Multimodal response not supported, expecting a string. Content type: object, Content:',
+        JSON.stringify({ type: 'image', data: 'base64data' }, null, 2),
+      );
+    });
+
+    it('returns success=false for array content and logs warning', async () => {
+      const mockResponse = new AIMessage(['text', { type: 'image', data: 'base64data' }] as any);
+      mockLLM.invoke.mockResolvedValue(mockResponse);
+
+      const messages = [{ role: 'user' as const, content: 'Hello' }];
+      const result = await provider.invokeModel(messages);
+
+      expect(result.metrics.success).toBe(false);
+      expect(result.message.content).toBe('');
+      expect(mockLogger.warn).toHaveBeenCalledWith(
+        'Multimodal response not supported, expecting a string. Content type: object, Content:',
+        JSON.stringify(['text', { type: 'image', data: 'base64data' }], null, 2),
+      );
+    });
+  });
+
   describe('mapProvider', () => {
     it('maps gemini to google-genai', () => {
       expect(LangChainProvider.mapProvider('gemini')).toBe('google-genai');

packages/ai-providers/server-ai-langchain/src/LangChainProvider.ts

@@ -50,6 +50,9 @@ export class LangChainProvider extends AIProvider {
     // Get the LangChain response
     const response: AIMessage = await this._llm.invoke(langchainMessages);
 
+    // Generate metrics early (assumes success by default)
+    const metrics = LangChainProvider.createAIMetrics(response);
+
     // Extract text content from the response
     let content: string = '';
     if (typeof response.content === 'string') {
@@ -60,6 +63,8 @@ export class LangChainProvider extends AIProvider {
         `Multimodal response not supported, expecting a string. Content type: ${typeof response.content}, Content:`,
         JSON.stringify(response.content, null, 2),
       );
+      // Update metrics to reflect content loss
+      metrics.success = false;
     }
 
     // Create the assistant message
@@ -68,9 +73,6 @@ export class LangChainProvider extends AIProvider {
       content,
     };
 
-    // Extract metrics including token usage and success status
-    const metrics = LangChainProvider.createAIMetrics(response);
-
     return {
       message: assistantMessage,
       metrics,
@@ -108,6 +110,7 @@ export class LangChainProvider extends AIProvider {
    * This method extracts token usage information and success status from LangChain responses
    * and returns a LaunchDarkly AIMetrics object.
    *
+   * @param langChainResponse The response from the LangChain model
    * @example
    * ```typescript
    * // Use with tracker.trackMetricsOf for automatic tracking
@@ -129,7 +132,7 @@ export class LangChainProvider extends AIProvider {
       };
     }
 
-    // LangChain responses that complete successfully are considered successful
+    // LangChain responses that complete successfully are considered successful by default
     return {
       success: true,
       usage,