Add 'Ask AI' capability to visualizations

CHANGELOG.md

@@ -4,6 +4,10 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ## [Unreleased]
 
+### 📈 Features/Enhancements
+
+ - Add 'Ask AI' capability to visualizations with screenshot capture and Claude 4 integration ([#TBD](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/TBD))
+
 ## [3.2.0-2025-08-06](https://github.com/opensearch-project/OpenSearch-Dashboards/releases/tag/3.2.0)
 
 ### 💥 Breaking Changes

package.json

@@ -266,6 +266,7 @@
     "globby": "^11.1.0",
     "handlebars": "4.7.7",
     "hjson": "3.2.1",
+    "html2canvas": "^1.4.1",
     "http-aws-es": "npm:@zhongnansu/http-aws-es@6.0.1",
     "http-proxy-agent": "^2.1.0",
     "https-proxy-agent": "^5.0.0",

packages/osd-agents/configuration/default_model.json

@@ -1,3 +1,3 @@
 {
-  "modelId": "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
+  "modelId": "us.anthropic.claude-sonnet-4-20250514-v1:0"
 }

packages/osd-agents/src/agents/langgraph/react_graph_nodes.ts

@@ -462,21 +462,94 @@ export class ReactGraphNodes {
         }
         return true;
       })
-      .map((msg) => ({
+      .map((msg) => {
         // Convert 'tool' role to 'user' role for Bedrock compatibility
         // Bedrock only accepts 'user' and 'assistant' roles
-        role: msg.role === 'tool' ? 'user' : msg.role || 'user',
+        const role = msg.role === 'tool' ? 'user' : msg.role || 'user';
+
+        let content: any[];
+
         // If content is already an array (proper format), use it directly
-        // This preserves toolUse and toolResult blocks
-        // Filter out empty text blocks to prevent ValidationException
-        content: Array.isArray(msg.content)
-          ? msg.content.filter((block: any) => !block.text || block.text.trim() !== '')
-          : [{ text: msg.content || '' }].filter((block: any) => block.text.trim() !== ''),
-      }));
+        if (Array.isArray(msg.content)) {
+          // Filter out empty text blocks to prevent ValidationException
+          content = msg.content.filter((block: any) => !block.text || block.text.trim() !== '');
+        } else {
+          // Convert string content to array format
+          const textContent = msg.content || '';
+          if (textContent.trim() !== '') {
+            content = [{ text: textContent }];
+          } else {
+            content = [];
+          }
+        }
+
+        // Handle image data for user messages
+        if (msg.role === 'user' && msg.imageData) {
+          this.logger.info('📸 Processing user message with image data', {
+            hasImageData: !!msg.imageData,
+            imageDataLength: msg.imageData.length,
+            contentBlocksCount: content.length,
+          });
+
+          try {
+            // Parse the base64 image data
+            // Expected format: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA..."
+            const imageDataMatch = msg.imageData.match(/^data:image\/([^;]+);base64,(.+)$/);
+
+            if (imageDataMatch) {
+              const [, format, base64Data] = imageDataMatch;
+
+              // Add image block to content array
+              const imageBlock = {
+                image: {
+                  format, // png, jpeg, gif, webp
+                  source: {
+                    bytes: Buffer.from(base64Data, 'base64'), // Convert base64 string to Buffer
+                  },
+                },
+              };
+
+              // Add image block to the beginning of content array
+              content.unshift(imageBlock);
+
+              this.logger.info('✅ Successfully added image block to message', {
+                format,
+                base64Length: base64Data.length,
+                bufferLength: Buffer.from(base64Data, 'base64').length,
+                totalContentBlocks: content.length,
+                imageBlockStructure: {
+                  hasImage: true,
+                  hasFormat: !!format,
+                  hasSource: true,
+                  hasBytesBuffer: Buffer.isBuffer(Buffer.from(base64Data, 'base64')),
+                },
+              });
+            } else {
+              this.logger.warn(
+                '⚠️ Invalid image data format, expected data:image/format;base64,data',
+                {
+                  imageDataPreview: msg.imageData.substring(0, 50) + '...',
+                }
+              );
+            }
+          } catch (error) {
+            this.logger.error('❌ Error processing image data', {
+              error: error instanceof Error ? error.message : String(error),
+              imageDataPreview: msg.imageData.substring(0, 50) + '...',
+            });
+          }
+        }
+
+        return {
+          role,
+          content,
+        };
+      });
 
     // Debug logging to catch toolUse/toolResult mismatch
     let toolUseCount = 0;
     let toolResultCount = 0;
+    let imageCount = 0;
 
     prepared.forEach((msg, index) => {
       if (msg.role === 'assistant' && Array.isArray(msg.content)) {
@@ -488,10 +561,15 @@ export class ReactGraphNodes {
       }
       if (msg.role === 'user' && Array.isArray(msg.content)) {
         const msgToolResults = msg.content.filter((c: any) => c.toolResult).length;
+        const msgImages = msg.content.filter((c: any) => c.image).length;
         toolResultCount += msgToolResults;
+        imageCount += msgImages;
         if (msgToolResults > 0) {
           this.logger.info(`Message ${index} (user): ${msgToolResults} toolResult blocks`);
         }
+        if (msgImages > 0) {
+          this.logger.info(`Message ${index} (user): ${msgImages} image blocks`);
+        }
       }
     });
 
@@ -504,6 +582,10 @@ export class ReactGraphNodes {
       });
     }
 
+    if (imageCount > 0) {
+      this.logger.info(`📊 Total images in conversation: ${imageCount}`);
+    }
+
     return prepared;
   }
 }

src/core/public/chat/chat_service.ts

@@ -153,6 +153,20 @@ export class ChatService implements CoreService<ChatServiceSetup, ChatServiceSta
         return this.implementation.sendMessageWithWindow(content, messages, options);
       },
 
+      setPendingImage: (imageData: string | undefined) => {
+        if (!this.implementation?.setPendingImage) {
+          return;
+        }
+        return this.implementation.setPendingImage(imageData);
+      },
+
+      setCapturingImage: (isCapturing: boolean) => {
+        if (!this.implementation?.setCapturingImage) {
+          return;
+        }
+        return this.implementation.setCapturingImage(isCapturing);
+      },
+
       // Infrastructure service - use getter to ensure dynamic access
       get suggestedActionsService() {
         return chatServiceInstance.suggestedActionsService;

src/core/public/chat/types.ts

@@ -58,11 +58,12 @@ export interface AssistantMessage extends BaseMessage {
 }
 
 /**
- * User message type
+ * User message type with optional image content
  */
 export interface UserMessage extends BaseMessage {
   role: 'user';
   content: string;
+  imageData?: string; // Base64 encoded image data
 }
 
 /**
@@ -134,13 +135,16 @@ export interface ChatServiceInterface {
   closeWindow(): Promise<void>;
   sendMessage(
     content: string,
-    messages: Message[]
+    messages: Message[],
+    imageData?: string
   ): Promise<{ observable: any; userMessage: UserMessage }>;
   sendMessageWithWindow(
     content: string,
     messages: Message[],
-    options?: { clearConversation?: boolean }
+    options?: { clearConversation?: boolean; imageData?: string }
   ): Promise<{ observable: any; userMessage: UserMessage }>;
+  setPendingImage?(imageData: string | undefined): void;
+  setCapturingImage?(isCapturing: boolean): void;
 }
 
 /**
@@ -150,18 +154,23 @@ export interface ChatImplementationFunctions {
   // Message operations
   sendMessage: (
     content: string,
-    messages: Message[]
+    messages: Message[],
+    imageData?: string
   ) => Promise<{ observable: any; userMessage: UserMessage }>;
 
   sendMessageWithWindow: (
     content: string,
     messages: Message[],
-    options?: { clearConversation?: boolean }
+    options?: { clearConversation?: boolean; imageData?: string }
   ) => Promise<{ observable: any; userMessage: UserMessage }>;
 
   // Window operations
   openWindow: () => Promise<void>;
   closeWindow: () => Promise<void>;
+
+  // Image operations
+  setPendingImage?: (imageData: string | undefined) => void;
+  setCapturingImage?: (isCapturing: boolean) => void;
 }
 
 /**

src/plugins/chat/common/types.ts

@@ -56,6 +56,7 @@ export const AssistantMessageSchema = BaseMessageSchema.extend({
 export const UserMessageSchema = BaseMessageSchema.extend({
   role: z.literal('user'),
   content: z.string(),
+  imageData: z.string().optional(), // Base64 encoded image data
 });
 
 export const ToolMessageSchema = z.object({

src/plugins/chat/public/components/chat_input.scss

@@ -1,12 +1,82 @@
+@import "@elastic/eui/src/global_styling/variables";
+@import "@elastic/eui/src/global_styling/mixins/shadow";
+
 .chatInput {
   grid-area: input;
   display: flex;
   flex-direction: column;
 
+  &__inputContainer {
+    display: flex;
+    flex-direction: column;
+    background-color: $euiColorEmptyShade;
+    border: 2px solid $euiColorLightShade;
+    border-radius: 12px;
+    padding: 8px;
+    transition: border-color 0.2s ease;
+
+    &:focus-within {
+      border-color: $euiColorPrimary;
+      box-shadow: 0 0 0 1px $euiColorPrimary;
+    }
+  }
+
   &__inputRow {
     display: grid;
     grid-template-columns: 1fr auto;
     gap: 8px;
     align-items: center;
   }
+
+  &__fieldWithImage {
+    border: none !important;
+    box-shadow: none !important;
+    background: transparent !important;
+
+    &:focus {
+      border: none !important;
+      box-shadow: none !important;
+    }
+  }
+
+  &__loadingIndicator {
+    display: flex;
+    align-items: center;
+    gap: 8px;
+    margin-bottom: 8px;
+    padding: 8px 12px;
+    background: $euiColorLightestShade;
+    border-radius: 8px;
+    border: 1px solid $euiColorLightShade;
+  }
+
+  &__imageAttachment {
+    position: relative;
+    display: inline-block;
+    margin-bottom: 8px;
+    padding: 4px;
+    background: white;
+    border-radius: 8px;
+    border: 1px solid $euiColorLightShade;
+
+    @include euiBottomShadowSmall;
+
+    max-width: 120px;
+  }
+
+  &__removeButton {
+    position: absolute;
+    top: -4px;
+    right: -4px;
+    background-color: $euiColorDanger;
+    border: 1px solid white;
+    border-radius: 50%;
+
+    @include euiBottomShadowSmall;
+
+    &:hover {
+      background-color: $euiColorDangerText;
+      transform: scale(1.1);
+    }
+  }
 }