diff --git a/CHANGELOG.md b/CHANGELOG.md index 32770dd4e994..207a0cb09ba8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## [Unreleased] +### 📈 Features/Enhancements + + - Add 'Ask AI' capability to visualizations with screenshot capture and Claude 4 integration ([#TBD](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/TBD)) + ## [3.2.0-2025-08-06](https://github.com/opensearch-project/OpenSearch-Dashboards/releases/tag/3.2.0) ### 💥 Breaking Changes diff --git a/package.json b/package.json index e3c7b9267050..8756013465f5 100644 --- a/package.json +++ b/package.json @@ -266,6 +266,7 @@ "globby": "^11.1.0", "handlebars": "4.7.7", "hjson": "3.2.1", + "html2canvas": "^1.4.1", "http-aws-es": "npm:@zhongnansu/http-aws-es@6.0.1", "http-proxy-agent": "^2.1.0", "https-proxy-agent": "^5.0.0", diff --git a/packages/osd-agents/configuration/default_model.json b/packages/osd-agents/configuration/default_model.json index 81a5222e475c..5d744e43265d 100644 --- a/packages/osd-agents/configuration/default_model.json +++ b/packages/osd-agents/configuration/default_model.json @@ -1,3 +1,3 @@ { - "modelId": "us.anthropic.claude-sonnet-4-5-20250929-v1:0" + "modelId": "us.anthropic.claude-sonnet-4-20250514-v1:0" } \ No newline at end of file diff --git a/packages/osd-agents/src/agents/langgraph/react_graph_nodes.ts b/packages/osd-agents/src/agents/langgraph/react_graph_nodes.ts index 97774d1877ef..bf607d6a3bd8 100644 --- a/packages/osd-agents/src/agents/langgraph/react_graph_nodes.ts +++ b/packages/osd-agents/src/agents/langgraph/react_graph_nodes.ts @@ -462,21 +462,94 @@ export class ReactGraphNodes { } return true; }) - .map((msg) => ({ + .map((msg) => { // Convert 'tool' role to 'user' role for Bedrock compatibility // Bedrock only accepts 'user' and 'assistant' roles - role: msg.role === 'tool' ? 'user' : msg.role || 'user', + const role = msg.role === 'tool' ? 'user' : msg.role || 'user'; + + let content: any[]; + // If content is already an array (proper format), use it directly - // This preserves toolUse and toolResult blocks - // Filter out empty text blocks to prevent ValidationException - content: Array.isArray(msg.content) - ? msg.content.filter((block: any) => !block.text || block.text.trim() !== '') - : [{ text: msg.content || '' }].filter((block: any) => block.text.trim() !== ''), - })); + if (Array.isArray(msg.content)) { + // Filter out empty text blocks to prevent ValidationException + content = msg.content.filter((block: any) => !block.text || block.text.trim() !== ''); + } else { + // Convert string content to array format + const textContent = msg.content || ''; + if (textContent.trim() !== '') { + content = [{ text: textContent }]; + } else { + content = []; + } + } + + // Handle image data for user messages + if (msg.role === 'user' && msg.imageData) { + this.logger.info('📸 Processing user message with image data', { + hasImageData: !!msg.imageData, + imageDataLength: msg.imageData.length, + contentBlocksCount: content.length, + }); + + try { + // Parse the base64 image data + // Expected format: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA..." + const imageDataMatch = msg.imageData.match(/^data:image\/([^;]+);base64,(.+)$/); + + if (imageDataMatch) { + const [, format, base64Data] = imageDataMatch; + + // Add image block to content array + const imageBlock = { + image: { + format, // png, jpeg, gif, webp + source: { + bytes: Buffer.from(base64Data, 'base64'), // Convert base64 string to Buffer + }, + }, + }; + + // Add image block to the beginning of content array + content.unshift(imageBlock); + + this.logger.info('✅ Successfully added image block to message', { + format, + base64Length: base64Data.length, + bufferLength: Buffer.from(base64Data, 'base64').length, + totalContentBlocks: content.length, + imageBlockStructure: { + hasImage: true, + hasFormat: !!format, + hasSource: true, + hasBytesBuffer: Buffer.isBuffer(Buffer.from(base64Data, 'base64')), + }, + }); + } else { + this.logger.warn( + '⚠️ Invalid image data format, expected data:image/format;base64,data', + { + imageDataPreview: msg.imageData.substring(0, 50) + '...', + } + ); + } + } catch (error) { + this.logger.error('❌ Error processing image data', { + error: error instanceof Error ? error.message : String(error), + imageDataPreview: msg.imageData.substring(0, 50) + '...', + }); + } + } + + return { + role, + content, + }; + }); // Debug logging to catch toolUse/toolResult mismatch let toolUseCount = 0; let toolResultCount = 0; + let imageCount = 0; prepared.forEach((msg, index) => { if (msg.role === 'assistant' && Array.isArray(msg.content)) { @@ -488,10 +561,15 @@ export class ReactGraphNodes { } if (msg.role === 'user' && Array.isArray(msg.content)) { const msgToolResults = msg.content.filter((c: any) => c.toolResult).length; + const msgImages = msg.content.filter((c: any) => c.image).length; toolResultCount += msgToolResults; + imageCount += msgImages; if (msgToolResults > 0) { this.logger.info(`Message ${index} (user): ${msgToolResults} toolResult blocks`); } + if (msgImages > 0) { + this.logger.info(`Message ${index} (user): ${msgImages} image blocks`); + } } }); @@ -504,6 +582,10 @@ export class ReactGraphNodes { }); } + if (imageCount > 0) { + this.logger.info(`📊 Total images in conversation: ${imageCount}`); + } + return prepared; } } diff --git a/src/core/public/chat/chat_service.ts b/src/core/public/chat/chat_service.ts index f5b5ae8fafcd..d198a0d7813f 100644 --- a/src/core/public/chat/chat_service.ts +++ b/src/core/public/chat/chat_service.ts @@ -153,6 +153,20 @@ export class ChatService implements CoreService { + if (!this.implementation?.setPendingImage) { + return; + } + return this.implementation.setPendingImage(imageData); + }, + + setCapturingImage: (isCapturing: boolean) => { + if (!this.implementation?.setCapturingImage) { + return; + } + return this.implementation.setCapturingImage(isCapturing); + }, + // Infrastructure service - use getter to ensure dynamic access get suggestedActionsService() { return chatServiceInstance.suggestedActionsService; diff --git a/src/core/public/chat/types.ts b/src/core/public/chat/types.ts index e7a58bc5e3f6..f9ed8091b7bc 100644 --- a/src/core/public/chat/types.ts +++ b/src/core/public/chat/types.ts @@ -58,11 +58,12 @@ export interface AssistantMessage extends BaseMessage { } /** - * User message type + * User message type with optional image content */ export interface UserMessage extends BaseMessage { role: 'user'; content: string; + imageData?: string; // Base64 encoded image data } /** @@ -134,13 +135,16 @@ export interface ChatServiceInterface { closeWindow(): Promise; sendMessage( content: string, - messages: Message[] + messages: Message[], + imageData?: string ): Promise<{ observable: any; userMessage: UserMessage }>; sendMessageWithWindow( content: string, messages: Message[], - options?: { clearConversation?: boolean } + options?: { clearConversation?: boolean; imageData?: string } ): Promise<{ observable: any; userMessage: UserMessage }>; + setPendingImage?(imageData: string | undefined): void; + setCapturingImage?(isCapturing: boolean): void; } /** @@ -150,18 +154,23 @@ export interface ChatImplementationFunctions { // Message operations sendMessage: ( content: string, - messages: Message[] + messages: Message[], + imageData?: string ) => Promise<{ observable: any; userMessage: UserMessage }>; sendMessageWithWindow: ( content: string, messages: Message[], - options?: { clearConversation?: boolean } + options?: { clearConversation?: boolean; imageData?: string } ) => Promise<{ observable: any; userMessage: UserMessage }>; // Window operations openWindow: () => Promise; closeWindow: () => Promise; + + // Image operations + setPendingImage?: (imageData: string | undefined) => void; + setCapturingImage?: (isCapturing: boolean) => void; } /** diff --git a/src/plugins/chat/common/types.ts b/src/plugins/chat/common/types.ts index d7eb4a636e25..9215c84a033c 100644 --- a/src/plugins/chat/common/types.ts +++ b/src/plugins/chat/common/types.ts @@ -56,6 +56,7 @@ export const AssistantMessageSchema = BaseMessageSchema.extend({ export const UserMessageSchema = BaseMessageSchema.extend({ role: z.literal('user'), content: z.string(), + imageData: z.string().optional(), // Base64 encoded image data }); export const ToolMessageSchema = z.object({ diff --git a/src/plugins/chat/public/components/chat_input.scss b/src/plugins/chat/public/components/chat_input.scss index 65eb94532694..cd4536dbc825 100644 --- a/src/plugins/chat/public/components/chat_input.scss +++ b/src/plugins/chat/public/components/chat_input.scss @@ -1,12 +1,82 @@ +@import "@elastic/eui/src/global_styling/variables"; +@import "@elastic/eui/src/global_styling/mixins/shadow"; + .chatInput { grid-area: input; display: flex; flex-direction: column; + &__inputContainer { + display: flex; + flex-direction: column; + background-color: $euiColorEmptyShade; + border: 2px solid $euiColorLightShade; + border-radius: 12px; + padding: 8px; + transition: border-color 0.2s ease; + + &:focus-within { + border-color: $euiColorPrimary; + box-shadow: 0 0 0 1px $euiColorPrimary; + } + } + &__inputRow { display: grid; grid-template-columns: 1fr auto; gap: 8px; align-items: center; } + + &__fieldWithImage { + border: none !important; + box-shadow: none !important; + background: transparent !important; + + &:focus { + border: none !important; + box-shadow: none !important; + } + } + + &__loadingIndicator { + display: flex; + align-items: center; + gap: 8px; + margin-bottom: 8px; + padding: 8px 12px; + background: $euiColorLightestShade; + border-radius: 8px; + border: 1px solid $euiColorLightShade; + } + + &__imageAttachment { + position: relative; + display: inline-block; + margin-bottom: 8px; + padding: 4px; + background: white; + border-radius: 8px; + border: 1px solid $euiColorLightShade; + + @include euiBottomShadowSmall; + + max-width: 120px; + } + + &__removeButton { + position: absolute; + top: -4px; + right: -4px; + background-color: $euiColorDanger; + border: 1px solid white; + border-radius: 50%; + + @include euiBottomShadowSmall; + + &:hover { + background-color: $euiColorDangerText; + transform: scale(1.1); + } + } } diff --git a/src/plugins/chat/public/components/chat_input.test.tsx b/src/plugins/chat/public/components/chat_input.test.tsx new file mode 100644 index 000000000000..9dc7123bc6e1 --- /dev/null +++ b/src/plugins/chat/public/components/chat_input.test.tsx @@ -0,0 +1,269 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { render, screen, fireEvent } from '@testing-library/react'; +import { ChatInput } from './chat_input'; +import { ChatLayoutMode } from './chat_header_button'; + +describe('ChatInput', () => { + const defaultProps = { + layoutMode: ChatLayoutMode.SIDECAR, + input: '', + isStreaming: false, + onInputChange: jest.fn(), + onSend: jest.fn(), + onKeyDown: jest.fn(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('basic functionality', () => { + it('should render input field with correct placeholder', () => { + render(); + + expect(screen.getByPlaceholderText('Type your message...')).toBeInTheDocument(); + }); + + it('should render send button', () => { + render(); + + expect(screen.getByLabelText('Send message')).toBeInTheDocument(); + }); + + it('should call onInputChange when typing', () => { + const onInputChange = jest.fn(); + render(); + + const input = screen.getByPlaceholderText('Type your message...'); + fireEvent.change(input, { target: { value: 'test message' } }); + + expect(onInputChange).toHaveBeenCalledWith('test message'); + }); + + it('should call onSend when send button is clicked', () => { + const onSend = jest.fn(); + render(); + + const sendButton = screen.getByLabelText('Send message'); + fireEvent.click(sendButton); + + expect(onSend).toHaveBeenCalled(); + }); + + it('should call onKeyDown when key is pressed', () => { + const onKeyDown = jest.fn(); + render(); + + const input = screen.getByPlaceholderText('Type your message...'); + fireEvent.keyDown(input, { key: 'Enter' }); + + expect(onKeyDown).toHaveBeenCalled(); + }); + }); + + describe('loading indicator functionality', () => { + it('should show loading indicator when isCapturingImage is true', () => { + render(); + + expect(screen.getByText('Capturing screenshot...')).toBeInTheDocument(); + expect(screen.getByRole('progressbar')).toBeInTheDocument(); // EuiLoadingSpinner + }); + + it('should not show loading indicator when isCapturingImage is false', () => { + render(); + + expect(screen.queryByText('Capturing screenshot...')).not.toBeInTheDocument(); + expect(screen.queryByRole('progressbar')).not.toBeInTheDocument(); + }); + + it('should not show loading indicator when isCapturingImage is undefined', () => { + render(); + + expect(screen.queryByText('Capturing screenshot...')).not.toBeInTheDocument(); + expect(screen.queryByRole('progressbar')).not.toBeInTheDocument(); + }); + + it('should show loading indicator above image attachment when both are present', () => { + const mockImageData = + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='; + + render( + + ); + + expect(screen.getByText('Capturing screenshot...')).toBeInTheDocument(); + expect(screen.getByAltText('Visualization screenshot')).toBeInTheDocument(); + }); + }); + + describe('image attachment functionality', () => { + const mockImageData = + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='; + + it('should show image attachment when pendingImage is provided', () => { + render( + + ); + + expect(screen.getByAltText('Visualization screenshot')).toBeInTheDocument(); + }); + + it('should not show image attachment when pendingImage is not provided', () => { + render(); + + expect(screen.queryByAltText('Visualization screenshot')).not.toBeInTheDocument(); + }); + + it('should show remove button when image is present and onRemoveImage is provided', () => { + const onRemoveImage = jest.fn(); + render( + + ); + + expect(screen.getByLabelText('Remove image')).toBeInTheDocument(); + }); + + it('should call onRemoveImage when remove button is clicked', () => { + const onRemoveImage = jest.fn(); + render( + + ); + + const removeButton = screen.getByLabelText('Remove image'); + fireEvent.click(removeButton); + + expect(onRemoveImage).toHaveBeenCalled(); + }); + + it('should not show remove button when onRemoveImage is not provided', () => { + render(); + + expect(screen.queryByLabelText('Remove image')).not.toBeInTheDocument(); + }); + + it('should change placeholder text when image is present', () => { + render( + + ); + + expect( + screen.getByPlaceholderText('Ask a question about the visualization...') + ).toBeInTheDocument(); + expect(screen.queryByPlaceholderText('Type your message...')).not.toBeInTheDocument(); + }); + + it('should apply special CSS class to input when image is present', () => { + render( + + ); + + const input = screen.getByPlaceholderText('Ask a question about the visualization...'); + expect(input).toHaveClass('chatInput__fieldWithImage'); + }); + }); + + describe('button states', () => { + it('should disable send button when input is empty and no image', () => { + render(); + + const sendButton = screen.getByLabelText('Send message'); + expect(sendButton).toBeDisabled(); + }); + + it('should enable send button when input has content', () => { + render(); + + const sendButton = screen.getByLabelText('Send message'); + expect(sendButton).not.toBeDisabled(); + }); + + it('should enable send button when image is present even without text', () => { + const mockImageData = 'data:image/png;base64,test'; + render( + + ); + + const sendButton = screen.getByLabelText('Send message'); + expect(sendButton).not.toBeDisabled(); + }); + + it('should disable send button when streaming', () => { + render(); + + const sendButton = screen.getByLabelText('Send message'); + expect(sendButton).toBeDisabled(); + }); + + it('should disable input field when streaming', () => { + render(); + + const input = screen.getByPlaceholderText('Type your message...'); + expect(input).toBeDisabled(); + }); + + it('should show different icon when streaming', () => { + const { rerender } = render(); + + // Check initial icon (sortUp) + let sendButton = screen.getByLabelText('Send message'); + expect(sendButton.querySelector('[data-euiicon-type="sortUp"]')).toBeInTheDocument(); + + // Check streaming icon (generate) + rerender(); + sendButton = screen.getByLabelText('Send message'); + expect(sendButton.querySelector('[data-euiicon-type="generate"]')).toBeInTheDocument(); + }); + }); + + describe('layout modes', () => { + it('should apply sidecar layout class', () => { + const { container } = render( + + ); + + expect(container.querySelector('.chatInput--sidecar')).toBeInTheDocument(); + }); + + it('should apply fullscreen layout class', () => { + const { container } = render( + + ); + + expect(container.querySelector('.chatInput--fullscreen')).toBeInTheDocument(); + }); + }); + + describe('accessibility', () => { + it('should have proper aria labels', () => { + const mockImageData = 'data:image/png;base64,test'; + render( + + ); + + expect(screen.getByLabelText('Send message')).toBeInTheDocument(); + expect(screen.getByLabelText('Remove image')).toBeInTheDocument(); + expect(screen.getByAltText('Visualization screenshot')).toBeInTheDocument(); + }); + + it('should have proper role for loading spinner', () => { + render(); + + expect(screen.getByRole('progressbar')).toBeInTheDocument(); + }); + }); +}); diff --git a/src/plugins/chat/public/components/chat_input.tsx b/src/plugins/chat/public/components/chat_input.tsx index 9030b0c0ad05..d4b07cc990af 100644 --- a/src/plugins/chat/public/components/chat_input.tsx +++ b/src/plugins/chat/public/components/chat_input.tsx @@ -4,7 +4,7 @@ */ import React from 'react'; -import { EuiFieldText, EuiButtonIcon } from '@elastic/eui'; +import { EuiFieldText, EuiButtonIcon, EuiImage, EuiLoadingSpinner, EuiText } from '@elastic/eui'; import { ChatLayoutMode } from './chat_header_button'; import { ContextPills } from './context_pills'; import './chat_input.scss'; @@ -13,40 +13,84 @@ interface ChatInputProps { layoutMode: ChatLayoutMode; input: string; isStreaming: boolean; + pendingImage?: string; // Base64 image data + isCapturingImage?: boolean; // New prop for screenshot loading state onInputChange: (value: string) => void; onSend: () => void; onKeyDown: (e: React.KeyboardEvent) => void; + onRemoveImage?: () => void; } export const ChatInput: React.FC = ({ layoutMode, input, isStreaming, + pendingImage, + isCapturingImage, onInputChange, onSend, onKeyDown, + onRemoveImage, }) => { return (
-
- onInputChange(e.target.value)} - onKeyDown={onKeyDown} - disabled={isStreaming} - fullWidth - /> - + +
+ {/* Loading indicator for screenshot capture */} + {isCapturingImage && ( +
+ + + Capturing screenshot... + +
+ )} + + {/* Image preview integrated into input */} + {pendingImage && ( +
+ + {onRemoveImage && ( + + )} +
+ )} + +
+ onInputChange(e.target.value)} + onKeyDown={onKeyDown} + disabled={isStreaming} + fullWidth + className={pendingImage ? 'chatInput__fieldWithImage' : ''} + /> + +
); diff --git a/src/plugins/chat/public/components/chat_input_loading.test.tsx b/src/plugins/chat/public/components/chat_input_loading.test.tsx new file mode 100644 index 000000000000..7182a4137573 --- /dev/null +++ b/src/plugins/chat/public/components/chat_input_loading.test.tsx @@ -0,0 +1,98 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { render, screen } from '@testing-library/react'; +import { ChatInput } from './chat_input'; +import { ChatLayoutMode } from './chat_header_button'; + +describe('ChatInput Loading Indicator', () => { + const defaultProps = { + layoutMode: ChatLayoutMode.SIDECAR, + input: '', + isStreaming: false, + onInputChange: jest.fn(), + onSend: jest.fn(), + onKeyDown: jest.fn(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('loading indicator functionality', () => { + it('should show loading indicator when isCapturingImage is true', () => { + render(); + + expect(screen.getByText('Capturing screenshot...')).toBeInTheDocument(); + expect(screen.getByTestId('euiLoadingSpinner')).toBeInTheDocument(); + }); + + it('should not show loading indicator when isCapturingImage is false', () => { + render(); + + expect(screen.queryByText('Capturing screenshot...')).not.toBeInTheDocument(); + expect(screen.queryByTestId('euiLoadingSpinner')).not.toBeInTheDocument(); + }); + + it('should not show loading indicator when isCapturingImage is undefined', () => { + render(); + + expect(screen.queryByText('Capturing screenshot...')).not.toBeInTheDocument(); + expect(screen.queryByTestId('euiLoadingSpinner')).not.toBeInTheDocument(); + }); + + it('should show loading indicator above image attachment when both are present', () => { + const mockImageData = 'data:image/png;base64,test'; + + render( + + ); + + expect(screen.getByText('Capturing screenshot...')).toBeInTheDocument(); + expect(screen.getByAltText('Visualization screenshot')).toBeInTheDocument(); + }); + }); + + describe('image attachment with loading states', () => { + const mockImageData = 'data:image/png;base64,test'; + + it('should change placeholder text when image is present', () => { + render( + + ); + + expect( + screen.getByPlaceholderText('Ask a question about the visualization...') + ).toBeInTheDocument(); + }); + + it('should enable send button when image is present even without text', () => { + render( + + ); + + const sendButton = screen.getByLabelText('Send message'); + expect(sendButton).not.toBeDisabled(); + }); + + it('should show loading indicator without image initially', () => { + render(); + + expect(screen.getByText('Capturing screenshot...')).toBeInTheDocument(); + expect(screen.queryByAltText('Visualization screenshot')).not.toBeInTheDocument(); + }); + }); +}); diff --git a/src/plugins/chat/public/components/chat_window.test.tsx b/src/plugins/chat/public/components/chat_window.test.tsx index db9b67a056f2..687d459403a5 100644 --- a/src/plugins/chat/public/components/chat_window.test.tsx +++ b/src/plugins/chat/public/components/chat_window.test.tsx @@ -41,11 +41,15 @@ describe('ChatWindow', () => { let mockCore: ReturnType; let mockContextProvider: any; let mockChatService: jest.Mocked; + let mockSuggestedActionsService: any; beforeEach(() => { jest.clearAllMocks(); mockCore = coreMock.createStart(); mockContextProvider = {}; + mockSuggestedActionsService = { + registerProvider: jest.fn(), + }; mockChatService = { sendMessage: jest.fn().mockResolvedValue({ observable: of({ type: 'message', content: 'test' }), @@ -63,7 +67,12 @@ describe('ChatWindow', () => { - {component} + + {component} + ); }; @@ -72,7 +81,7 @@ describe('ChatWindow', () => { it('should expose startNewChat method via ref', () => { const ref = React.createRef(); - renderWithContext(); + renderWithContext(); expect(ref.current).toBeDefined(); expect(ref.current?.startNewChat).toBeDefined(); @@ -82,17 +91,37 @@ describe('ChatWindow', () => { it('should expose sendMessage method via ref', () => { const ref = React.createRef(); - renderWithContext(); + renderWithContext(); expect(ref.current).toBeDefined(); expect(ref.current?.sendMessage).toBeDefined(); expect(typeof ref.current?.sendMessage).toBe('function'); }); + it('should expose setPendingImage method via ref', () => { + const ref = React.createRef(); + + renderWithContext(); + + expect(ref.current).toBeDefined(); + expect(ref.current?.setPendingImage).toBeDefined(); + expect(typeof ref.current?.setPendingImage).toBe('function'); + }); + + it('should expose setCapturingImage method via ref', () => { + const ref = React.createRef(); + + renderWithContext(); + + expect(ref.current).toBeDefined(); + expect(ref.current?.setCapturingImage).toBeDefined(); + expect(typeof ref.current?.setCapturingImage).toBe('function'); + }); + it('should call chatService.newThread when startNewChat is invoked', () => { const ref = React.createRef(); - renderWithContext(); + renderWithContext(); ref.current?.startNewChat(); @@ -102,7 +131,7 @@ describe('ChatWindow', () => { it('should call chatService.sendMessage when sendMessage is invoked via ref', async () => { const ref = React.createRef(); - renderWithContext(); + renderWithContext(); // Wait for the sendMessage to complete await ref.current?.sendMessage({ content: 'test message from ref' }); @@ -117,15 +146,224 @@ describe('ChatWindow', () => { }); }); + describe('image capture functionality', () => { + it('should pass isCapturingImage state to ChatInput', () => { + const ref = React.createRef(); + + renderWithContext(); + + // Initially should be false + expect(ref.current?.setCapturingImage).toBeDefined(); + + // Test that the method exists and can be called + expect(() => { + ref.current?.setCapturingImage(true); + }).not.toThrow(); + + expect(() => { + ref.current?.setCapturingImage(false); + }).not.toThrow(); + }); + + it('should manage pendingImage state correctly', () => { + const ref = React.createRef(); + + renderWithContext(); + + // Test that setPendingImage method exists and can be called + expect(() => { + ref.current?.setPendingImage('data:image/png;base64,test'); + }).not.toThrow(); + + expect(() => { + ref.current?.setPendingImage(undefined); + }).not.toThrow(); + }); + + it('should clear capturing state on new chat', () => { + const ref = React.createRef(); + + renderWithContext(); + + // Set capturing state + ref.current?.setCapturingImage(true); + + // Start new chat should clear the state + ref.current?.startNewChat(); + + expect(mockChatService.newThread).toHaveBeenCalled(); + }); + + it('should handle image data in sendMessage', async () => { + const ref = React.createRef(); + + renderWithContext(); + + const imageData = 'data:image/png;base64,test'; + + // Send message with image data + await ref.current?.sendMessage({ + content: 'test message', + imageData, + }); + + // Wait for any pending promises to resolve + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(mockChatService.sendMessage).toHaveBeenCalledWith( + 'test message', + expect.any(Array), + imageData + ); + }); + + it('should send default message when only image is provided', async () => { + const ref = React.createRef(); + + renderWithContext(); + + const imageData = 'data:image/png;base64,test'; + + // Send message with only image data (no content) + await ref.current?.sendMessage({ + content: '', + imageData, + }); + + // Wait for any pending promises to resolve + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(mockChatService.sendMessage).toHaveBeenCalledWith( + 'Can you analyze this visualization?', + expect.any(Array), + imageData + ); + }); + }); + + describe('loading message functionality', () => { + it('should add loading message to timeline when sending a message', async () => { + renderWithContext(); + + // Mock the sendMessage to return a controllable observable + const loadingObservable = { + subscribe: jest.fn(() => { + // Don't call next immediately to simulate loading state + return { unsubscribe: jest.fn() }; + }), + }; + + mockChatService.sendMessage.mockResolvedValue({ + observable: loadingObservable, + userMessage: { id: 'user-1', content: 'test', role: 'user' }, + }); + + const ref = React.createRef(); + renderWithContext(); + + // Send a message + await ref.current?.sendMessage({ content: 'test message' }); + + // Wait for state updates + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(mockChatService.sendMessage).toHaveBeenCalled(); + expect(loadingObservable.subscribe).toHaveBeenCalled(); + }); + + it('should remove loading message when first response is received', async () => { + const responseObservable = { + subscribe: jest.fn((callbacks) => { + // Simulate receiving a response + setTimeout(() => { + callbacks.next({ type: 'message', content: 'response' }); + }, 10); + return { unsubscribe: jest.fn() }; + }), + }; + + mockChatService.sendMessage.mockResolvedValue({ + observable: responseObservable, + userMessage: { id: 'user-1', content: 'test', role: 'user' }, + }); + + const ref = React.createRef(); + renderWithContext(); + + // Send a message + await ref.current?.sendMessage({ content: 'test message' }); + + // Wait for the response to be processed + await new Promise((resolve) => setTimeout(resolve, 20)); + + expect(responseObservable.subscribe).toHaveBeenCalled(); + }); + + it('should remove loading message on error', async () => { + const errorObservable = { + subscribe: jest.fn((callbacks) => { + // Simulate an error + setTimeout(() => { + callbacks.error(new Error('Test error')); + }, 10); + return { unsubscribe: jest.fn() }; + }), + }; + + mockChatService.sendMessage.mockResolvedValue({ + observable: errorObservable, + userMessage: { id: 'user-1', content: 'test', role: 'user' }, + }); + + const ref = React.createRef(); + renderWithContext(); + + // Send a message + await ref.current?.sendMessage({ content: 'test message' }); + + // Wait for the error to be processed + await new Promise((resolve) => setTimeout(resolve, 20)); + + expect(errorObservable.subscribe).toHaveBeenCalled(); + }); + + it('should remove loading message on completion', async () => { + const completionObservable = { + subscribe: jest.fn((callbacks) => { + // Simulate completion without response + setTimeout(() => { + callbacks.complete(); + }, 10); + return { unsubscribe: jest.fn() }; + }), + }; + + mockChatService.sendMessage.mockResolvedValue({ + observable: completionObservable, + userMessage: { id: 'user-1', content: 'test', role: 'user' }, + }); + + const ref = React.createRef(); + renderWithContext(); + + // Send a message + await ref.current?.sendMessage({ content: 'test message' }); + + // Wait for completion + await new Promise((resolve) => setTimeout(resolve, 20)); + + expect(completionObservable.subscribe).toHaveBeenCalled(); + }); + }); describe('persistence integration', () => { it('should restore timeline from persisted messages on mount', () => { const persistedMessages = [ - { id: '1', role: 'user', content: 'Hello' }, - { id: '2', role: 'assistant', content: 'Hi there!' }, + { id: '1', role: 'user', content: 'Hello' } as const, + { id: '2', role: 'assistant', content: 'Hi there!' } as const, ]; mockChatService.getCurrentMessages.mockReturnValue(persistedMessages); - renderWithContext(); + renderWithContext(); // Should call getCurrentMessages on mount expect(mockChatService.getCurrentMessages).toHaveBeenCalled(); @@ -134,38 +372,24 @@ describe('ChatWindow', () => { it('should not restore timeline when no persisted messages exist', () => { mockChatService.getCurrentMessages.mockReturnValue([]); - renderWithContext(); + renderWithContext(); // Should call getCurrentMessages but timeline should remain empty expect(mockChatService.getCurrentMessages).toHaveBeenCalled(); }); it('should sync timeline changes with ChatService for persistence', async () => { - const { rerender } = renderWithContext(); + renderWithContext(); // Wait for initial render and useEffect calls await new Promise((resolve) => setTimeout(resolve, 0)); // Initially called with empty timeline expect(mockChatService.updateCurrentMessages).toHaveBeenCalledWith([]); - - // Simulate timeline change by re-rendering - rerender( - - - - - - ); - - // Should call updateCurrentMessages whenever timeline changes - expect(mockChatService.updateCurrentMessages).toHaveBeenCalled(); }); it('should call updateCurrentMessages on every timeline update', async () => { - renderWithContext(); + renderWithContext(); // Wait for initial mount effects await new Promise((resolve) => setTimeout(resolve, 0)); diff --git a/src/plugins/chat/public/components/chat_window.tsx b/src/plugins/chat/public/components/chat_window.tsx index a5d261a3014c..bc998b499455 100644 --- a/src/plugins/chat/public/components/chat_window.tsx +++ b/src/plugins/chat/public/components/chat_window.tsx @@ -6,12 +6,9 @@ /* eslint-disable no-console */ import React, { useState, useEffect, useMemo, useImperativeHandle, useCallback, useRef } from 'react'; -import { CoreStart } from '../../../../core/public'; import { useChatContext } from '../contexts/chat_context'; import { ChatEventHandler } from '../services/chat_event_handler'; -import { useOpenSearchDashboards } from '../../../opensearch_dashboards_react/public'; import { - ContextProviderStart, AssistantActionService, } from '../../../context_provider/public'; import { @@ -30,7 +27,9 @@ import { ChatInput } from './chat_input'; export interface ChatWindowInstance{ startNewChat: ()=>void; - sendMessage: (options:{content: string})=>Promise; + sendMessage: (options:{content: string; imageData?: string})=>Promise; + setPendingImage: (imageData: string | undefined) => void; + setCapturingImage: (isCapturing: boolean) => void; } interface ChatWindowProps { @@ -54,14 +53,12 @@ const ChatWindowContent = React.forwardRef( const service = AssistantActionService.getInstance(); const { chatService } = useChatContext(); - const { services } = useOpenSearchDashboards<{ - core: CoreStart; - contextProvider?: ContextProviderStart; - }>(); const [timeline, setTimeline] = useState([]); const [input, setInput] = useState(''); const [isStreaming, setIsStreaming] = useState(false); const [currentRunId, setCurrentRunId] = useState(null); + const [pendingImage, setPendingImage] = useState(undefined); + const [isCapturingImage, setIsCapturingImage] = useState(false); const handleSendRef = useRef(); const timelineRef = React.useRef(timeline); @@ -115,17 +112,27 @@ const ChatWindowContent = React.forwardRef( chatService.updateCurrentMessages(timeline); }, [timeline, chatService]); - const handleSend = async (options?: {input?: string}) => { + const handleSend = async (options?: {input?: string; imageData?: string}) => { const messageContent = options?.input ?? input.trim(); - if (!messageContent || isStreaming) return; + + // Debug: Log image data flow + const imageToSend = options?.imageData ?? pendingImage; + + // Allow sending if there's either text content or an image + if ((!messageContent && !imageToSend) || isStreaming) return; + + // If no text content but there's an image, use a default message + const finalMessageContent = messageContent || 'Can you analyze this visualization?'; setInput(''); + setPendingImage(undefined); // Clear pending image after sending setIsStreaming(true); try { const { observable, userMessage } = await chatService.sendMessage( - messageContent, - timeline + finalMessageContent, + timeline, + imageToSend || undefined ); // Add user message immediately to timeline @@ -133,6 +140,7 @@ const ChatWindowContent = React.forwardRef( id: userMessage.id, role: 'user', content: userMessage.content, + imageData: userMessage.imageData, // Include image data in timeline }; setTimeline((prev) => [...prev, timelineUserMessage]); @@ -239,8 +247,16 @@ const ChatWindowContent = React.forwardRef( setTimeline([]); setCurrentRunId(null); setIsStreaming(false); + setPendingImage(undefined); // Clear pending image on new chat + setIsCapturingImage(false); // Clear capturing state on new chat }, [chatService]); + const handleRemoveImage = () => { + setPendingImage(undefined); + }; + + + const currentState = service.getCurrentState(); const enhancedProps = { toolCallStates: currentState.toolCallStates, @@ -249,7 +265,9 @@ const ChatWindowContent = React.forwardRef( useImperativeHandle(ref, ()=>({ startNewChat: ()=>handleNewChat(), - sendMessage: async ({content})=>(await handleSendRef.current?.({input:content})) + sendMessage: async ({content, imageData})=>(await handleSendRef.current?.({input:content, imageData})), + setPendingImage: (imageData: string | undefined) => setPendingImage(imageData), + setCapturingImage: (isCapturing: boolean) => setIsCapturingImage(isCapturing) }), [handleNewChat]); return ( @@ -270,13 +288,18 @@ const ChatWindowContent = React.forwardRef( {...enhancedProps} /> + + ); diff --git a/src/plugins/chat/public/components/chat_window_loading.test.tsx b/src/plugins/chat/public/components/chat_window_loading.test.tsx new file mode 100644 index 000000000000..ce846366bd61 --- /dev/null +++ b/src/plugins/chat/public/components/chat_window_loading.test.tsx @@ -0,0 +1,167 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { render, act } from '@testing-library/react'; +import { ChatWindow, ChatWindowInstance } from './chat_window'; +import { coreMock } from '../../../../core/public/mocks'; +import { of } from 'rxjs'; +import { OpenSearchDashboardsContextProvider } from '../../../opensearch_dashboards_react/public'; +import { ChatProvider } from '../contexts/chat_context'; +import { ChatService } from '../services/chat_service'; + +// Create mock observable +const mockObservable = of({ toolDefinitions: [], toolCallStates: {} }); + +// Mock dependencies +jest.mock('../../../context_provider/public', () => ({ + AssistantActionService: { + getInstance: jest.fn(() => ({ + getState$: jest.fn(() => mockObservable), + getCurrentState: jest.fn(() => ({ toolDefinitions: [], toolCallStates: {} })), + getActionRenderer: jest.fn(), + })), + }, +})); + +jest.mock('../services/chat_event_handler', () => ({ + ChatEventHandler: jest.fn().mockImplementation(() => ({ + handleEvent: jest.fn(), + clearState: jest.fn(), + })), +})); + +describe('ChatWindow Loading Functionality', () => { + let mockCore: ReturnType; + let mockContextProvider: any; + let mockChatService: jest.Mocked; + let mockSuggestedActionsService: any; + + beforeEach(() => { + jest.clearAllMocks(); + mockCore = coreMock.createStart(); + mockContextProvider = {}; + mockSuggestedActionsService = { + registerProvider: jest.fn(), + }; + mockChatService = { + sendMessage: jest.fn().mockResolvedValue({ + observable: of({ type: 'message', content: 'test' }), + userMessage: { id: '1', content: 'test', role: 'user' }, + }), + newThread: jest.fn(), + getCurrentMessages: jest.fn().mockReturnValue([]), + updateCurrentMessages: jest.fn(), + getThreadId: jest.fn().mockReturnValue('mock-thread-id'), + } as any; + }); + + const renderWithContext = (component: React.ReactElement) => { + return render( + + + {component} + + + ); + }; + + describe('image capture functionality', () => { + it('should expose setCapturingImage method via ref', () => { + const ref = React.createRef(); + + renderWithContext(); + + expect(ref.current).toBeDefined(); + expect(ref.current?.setCapturingImage).toBeDefined(); + expect(typeof ref.current?.setCapturingImage).toBe('function'); + }); + + it('should expose setPendingImage method via ref', () => { + const ref = React.createRef(); + + renderWithContext(); + + expect(ref.current).toBeDefined(); + expect(ref.current?.setPendingImage).toBeDefined(); + expect(typeof ref.current?.setPendingImage).toBe('function'); + }); + + it('should allow calling setCapturingImage without errors', () => { + const ref = React.createRef(); + + renderWithContext(); + + expect(() => { + ref.current?.setCapturingImage(true); + }).not.toThrow(); + + expect(() => { + ref.current?.setCapturingImage(false); + }).not.toThrow(); + }); + + it('should allow calling setPendingImage without errors', () => { + const ref = React.createRef(); + + renderWithContext(); + + expect(() => { + ref.current?.setPendingImage('data:image/png;base64,test'); + }).not.toThrow(); + + expect(() => { + ref.current?.setPendingImage(undefined); + }).not.toThrow(); + }); + + it('should handle image data in sendMessage', async () => { + const ref = React.createRef(); + + renderWithContext(); + + const imageData = 'data:image/png;base64,test'; + + await act(async () => { + await ref.current?.sendMessage({ + content: 'test message', + imageData, + }); + }); + + expect(mockChatService.sendMessage).toHaveBeenCalledWith( + 'test message', + expect.any(Array), + imageData + ); + }); + + it('should send default message when only image is provided', async () => { + const ref = React.createRef(); + + renderWithContext(); + + const imageData = 'data:image/png;base64,test'; + + await act(async () => { + await ref.current?.sendMessage({ + content: '', + imageData, + }); + }); + + expect(mockChatService.sendMessage).toHaveBeenCalledWith( + 'Can you analyze this visualization?', + expect.any(Array), + imageData + ); + }); + }); +}); diff --git a/src/plugins/chat/public/components/message_row.scss b/src/plugins/chat/public/components/message_row.scss index 7795cf4a9a48..0ab7b3f8eb74 100644 --- a/src/plugins/chat/public/components/message_row.scss +++ b/src/plugins/chat/public/components/message_row.scss @@ -42,6 +42,10 @@ } } + &__image { + margin-bottom: 12px; + } + &__actions { position: absolute; top: 8px; diff --git a/src/plugins/chat/public/components/message_row.tsx b/src/plugins/chat/public/components/message_row.tsx index c8e60246e017..dda539828478 100644 --- a/src/plugins/chat/public/components/message_row.tsx +++ b/src/plugins/chat/public/components/message_row.tsx @@ -4,9 +4,9 @@ */ import React, { useState } from 'react'; -import { EuiPanel, EuiIcon, EuiButtonIcon } from '@elastic/eui'; +import { EuiPanel, EuiIcon, EuiButtonIcon, EuiImage } from '@elastic/eui'; import { Markdown } from '../../../opensearch_dashboards_react/public'; -import type { Message } from '../../common/types'; +import type { Message, UserMessage } from '../../common/types'; import './message_row.scss'; interface MessageRowProps { @@ -31,6 +31,10 @@ export const MessageRow: React.FC = ({ // Handle optional content const content = message.content || ''; + // Check if this is a user message with image data + const userMessage = message as UserMessage; + const hasImage = message.role === 'user' && userMessage.imageData; + return (
= ({
+ {/* Display image if present */} + {hasImage && userMessage.imageData && ( +
+ +
+ )}
{isStreaming && |} diff --git a/src/plugins/chat/public/plugin.ts b/src/plugins/chat/public/plugin.ts index cda92c833398..847293e9d0e4 100644 --- a/src/plugins/chat/public/plugin.ts +++ b/src/plugins/chat/public/plugin.ts @@ -125,6 +125,8 @@ export class ChatPlugin implements Plugin { sendMessageWithWindow: this.chatService.sendMessageWithWindow.bind(this.chatService), openWindow: this.chatService.openWindow.bind(this.chatService), closeWindow: this.chatService.closeWindow.bind(this.chatService), + setPendingImage: this.chatService.setPendingImage.bind(this.chatService), + setCapturingImage: this.chatService.setCapturingImage.bind(this.chatService), }); } diff --git a/src/plugins/chat/public/services/chat_service.ts b/src/plugins/chat/public/services/chat_service.ts index 81f888090746..034c201831b1 100644 --- a/src/plugins/chat/public/services/chat_service.ts +++ b/src/plugins/chat/public/services/chat_service.ts @@ -212,6 +212,18 @@ export class ChatService { this.chatWindowRef = null; } + public setPendingImage(imageData: string | undefined): void { + if (this.chatWindowRef?.current) { + this.chatWindowRef.current.setPendingImage(imageData); + } + } + + public setCapturingImage(isCapturing: boolean): void { + if (this.chatWindowRef?.current) { + this.chatWindowRef.current.setCapturingImage(isCapturing); + } + } + public async openWindow(): Promise { if (!this.coreChatService) { throw new Error('Core chat service not available'); @@ -229,7 +241,7 @@ export class ChatService { public async sendMessageWithWindow( content: string, messages: Message[], - options?: { clearConversation?: boolean } + options?: { clearConversation?: boolean; imageData?: string } ): Promise<{ observable: any; userMessage: UserMessage; @@ -250,13 +262,14 @@ export class ChatService { // If ChatWindow is available, delegate to its sendMessage for proper timeline management if (this.chatWindowRef?.current && this.isWindowOpen()) { try { - await this.chatWindowRef.current.sendMessage({ content }); + await this.chatWindowRef.current.sendMessage({ content, imageData: options?.imageData }); // Create a user message for consistency with the return type const userMessage: UserMessage = { id: this.generateMessageId(), role: 'user', content: content.trim(), + imageData: options?.imageData, }; // Return a dummy observable since ChatWindow handles everything internally @@ -271,7 +284,7 @@ export class ChatService { } // Fallback to direct service call - const result = await this.sendMessage(content, messages); + const result = await this.sendMessage(content, messages, options?.imageData); return result; } @@ -353,7 +366,8 @@ export class ChatService { public async sendMessage( content: string, - messages: Message[] + messages: Message[], + imageData?: string ): Promise<{ observable: any; userMessage: UserMessage; @@ -365,6 +379,7 @@ export class ChatService { id: this.generateMessageId(), role: 'user', content: content.trim(), + imageData, // Include image data if provided }; // Get workspace-aware data source ID diff --git a/src/plugins/visualizations/public/actions/ask_ai_action.test.tsx b/src/plugins/visualizations/public/actions/ask_ai_action.test.tsx new file mode 100644 index 000000000000..a83f00505cbc --- /dev/null +++ b/src/plugins/visualizations/public/actions/ask_ai_action.test.tsx @@ -0,0 +1,156 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { AskAIAction, ACTION_ASK_AI } from './ask_ai_action'; +import { coreMock } from '../../../../core/public/mocks'; +import { IEmbeddable } from '../../../embeddable/public'; + +describe('AskAIAction', () => { + let action: AskAIAction; + let mockCore: ReturnType; + let mockEmbeddable: jest.Mocked; + let mockChatService: any; + + beforeEach(() => { + jest.clearAllMocks(); + + mockCore = coreMock.createStart(); + + mockChatService = { + isAvailable: jest.fn(() => true), + openWindow: jest.fn(() => Promise.resolve()), + setPendingImage: jest.fn(), + setCapturingImage: jest.fn(), + }; + + mockCore.chat = mockChatService; + + mockEmbeddable = { + id: 'test-embeddable-id', + type: 'visualization', + getTitle: jest.fn(() => 'Test Visualization'), + getInput: jest.fn(), + getOutput: jest.fn(), + reload: jest.fn(), + destroy: jest.fn(), + updateInput: jest.fn(), + render: jest.fn(), + isContainer: false, + } as any; + + action = new AskAIAction(mockCore); + }); + + describe('basic properties', () => { + it('should have correct type and id', () => { + expect(action.type).toBe(ACTION_ASK_AI); + expect(action.id).toBe(ACTION_ASK_AI); + }); + + it('should have correct order', () => { + expect(action.order).toBe(100); + }); + + it('should return correct display name', () => { + const displayName = action.getDisplayName({ embeddable: mockEmbeddable }); + expect(displayName).toBe('Ask AI'); + }); + + it('should return correct icon type', () => { + const iconType = action.getIconType({ embeddable: mockEmbeddable }); + expect(iconType).toBe('discuss'); + }); + }); + + describe('compatibility checks', () => { + it('should be compatible with visualization embeddables when chat service is available', async () => { + const isCompatible = await action.isCompatible({ embeddable: mockEmbeddable }); + expect(isCompatible).toBe(true); + }); + + it('should not be compatible when chat service is not available', async () => { + mockChatService.isAvailable.mockReturnValue(false); + + const isCompatible = await action.isCompatible({ embeddable: mockEmbeddable }); + expect(isCompatible).toBe(false); + }); + + it('should not be compatible with error embeddables', async () => { + const errorEmbeddable = { + ...mockEmbeddable, + error: new Error('Test error'), + } as any; + + const isCompatible = await action.isCompatible({ embeddable: errorEmbeddable }); + expect(isCompatible).toBe(false); + }); + + it('should be compatible with various visualization types', async () => { + const visualizationTypes = [ + 'visualization', + 'lens', + 'map', + 'vega', + 'timelion', + 'input_control_vis', + 'metrics', + 'tagcloud', + 'region_map', + 'tile_map', + 'histogram', + 'line', + 'area', + 'bar', + 'pie', + 'metric', + 'table', + 'gauge', + 'goal', + 'heatmap', + ]; + + for (const type of visualizationTypes) { + const embeddable = { ...mockEmbeddable, type }; + const isCompatible = await action.isCompatible({ embeddable }); + expect(isCompatible).toBe(true); + } + }); + + it('should not be compatible with non-visualization types', async () => { + const nonVisualizationEmbeddable = { + ...mockEmbeddable, + type: 'dashboard', + }; + + const isCompatible = await action.isCompatible({ embeddable: nonVisualizationEmbeddable }); + expect(isCompatible).toBe(false); + }); + }); + + describe('error handling', () => { + it('should show warning when chat service is unavailable', async () => { + const coreWithoutChat = { ...mockCore, chat: undefined as any }; + const actionWithoutChat = new AskAIAction(coreWithoutChat); + + await actionWithoutChat.execute({ embeddable: mockEmbeddable }); + + expect(coreWithoutChat.notifications.toasts.addWarning).toHaveBeenCalledWith({ + title: 'Chat service unavailable', + text: 'The AI chat service is not available. Please check your configuration.', + }); + }); + + it('should show warning when chat service isAvailable returns false', async () => { + mockChatService.isAvailable.mockReturnValue(false); + + await action.execute({ embeddable: mockEmbeddable }); + + expect(mockCore.notifications.toasts.addWarning).toHaveBeenCalledWith({ + title: 'Chat service unavailable', + text: 'The AI chat service is not available. Please check your configuration.', + }); + }); + }); +}); diff --git a/src/plugins/visualizations/public/actions/ask_ai_action.tsx b/src/plugins/visualizations/public/actions/ask_ai_action.tsx new file mode 100644 index 000000000000..6e37aeba5dfd --- /dev/null +++ b/src/plugins/visualizations/public/actions/ask_ai_action.tsx @@ -0,0 +1,337 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { i18n } from '@osd/i18n'; +import { EuiIconType } from '@elastic/eui/src/components/icon/icon'; +import { ActionByType, IncompatibleActionError } from '../../../ui_actions/public'; +import { IEmbeddable, isErrorEmbeddable } from '../../../embeddable/public'; +import { CoreStart } from '../../../../core/public'; + +export const ACTION_ASK_AI = 'askAI'; + +export interface AskAIActionContext { + embeddable: IEmbeddable; +} + +/** + * Captures a screenshot of the visualization element + */ +async function captureVisualizationScreenshot(embeddable: IEmbeddable): Promise { + try { + let targetElement: HTMLElement | null = null; + let strategyUsed = ''; + + // Strategy 1: Look for embeddable by data-embeddable-id (most reliable) + targetElement = document.querySelector( + `[data-embeddable-id="${embeddable.id}"]` + ) as HTMLElement; + if (targetElement) { + strategyUsed = 'data-embeddable-id'; + } + + // Strategy 2: Look for embeddable panel by test subject + if (!targetElement) { + const headingElement = document.querySelector( + `[data-test-subj="embeddablePanelHeading-${embeddable.id}"]` + ) as HTMLElement; + if (headingElement) { + targetElement = headingElement.closest('.embPanel') as HTMLElement; + if (targetElement) { + strategyUsed = 'embeddablePanelHeading'; + } + } + } + + // Strategy 3: Look for embeddable by panel action button (more specific) + if (!targetElement) { + // Look for any Ask AI action button that might have been clicked + const actionButtons = document.querySelectorAll( + `[data-test-subj*="embeddablePanelAction-${ACTION_ASK_AI}"]` + ); + + for (let i = 0; i < actionButtons.length; i++) { + const button = actionButtons[i]; + const panel = button.closest('.embPanel') as HTMLElement; + if (panel) { + // Check if this panel contains our embeddable ID + const panelEmbeddableId = panel + .querySelector('[data-embeddable-id]') + ?.getAttribute('data-embeddable-id'); + if (panelEmbeddableId === embeddable.id) { + targetElement = panel; + strategyUsed = 'actionButton'; + break; + } + } + } + } + + // Strategy 4: Use embeddable's DOM node if available + if (!targetElement && (embeddable as any).getRoot) { + try { + const root = (embeddable as any).getRoot(); + if (root && root.domNode) { + targetElement = root.domNode as HTMLElement; + strategyUsed = 'embeddable.root.domNode'; + } else if (root && (root as any).getContainer && (root as any).getContainer().node) { + const containerNode = (root as any).getContainer().node; + targetElement = + (containerNode.querySelector( + `[data-embeddable-id="${embeddable.id}"]` + ) as HTMLElement) || containerNode; + strategyUsed = 'embeddable.root.container'; + } + } catch (error) { + // Ignore errors accessing embeddable root + } + } + + // Strategy 5: Direct DOM node access + if (!targetElement && (embeddable as any).domNode) { + targetElement = (embeddable as any).domNode as HTMLElement; + strategyUsed = 'embeddable.domNode'; + } + + // Strategy 6: Search by embeddable ID in HTML content (less reliable) + if (!targetElement) { + const embeddablePanels = document.querySelectorAll( + '.embPanel, .embeddable, [class*="embeddable"]' + ); + + for (let i = 0; i < embeddablePanels.length; i++) { + const panel = embeddablePanels[i]; + const panelHtml = panel.outerHTML; + if (panelHtml.includes(embeddable.id)) { + targetElement = panel as HTMLElement; + strategyUsed = 'htmlContent'; + break; + } + } + } + + // Strategy 7: Find by title (least reliable, may match wrong visualization) + if (!targetElement) { + const title = embeddable.getTitle(); + if (title) { + const titleElements = document.querySelectorAll( + `[title*="${title}"], [aria-label*="${title}"]` + ); + + for (let i = 0; i < titleElements.length; i++) { + const titleEl = titleElements[i]; + const panel = titleEl.closest( + '.embPanel, .embeddable, [class*="embeddable"]' + ) as HTMLElement; + if (panel) { + targetElement = panel; + strategyUsed = 'title'; + break; + } + } + } + } + + if (!targetElement) { + return null; + } + + // Find the visualization content within the target element + const visualizationSelectors = [ + '.visualization', + '.visChart', + '.vis-container', + '.lnsVisualizationContainer', + '.vgaVis', + '.mapContainer', + '.embPanel__content .visualization', + '.embPanel__content', + 'canvas', + 'svg', + '.chart-container', + '.vis-wrapper', + ]; + + let visualizationElement: HTMLElement | null = null; + let selectorUsed = ''; + + for (const selector of visualizationSelectors) { + visualizationElement = targetElement.querySelector(selector) as HTMLElement; + if (visualizationElement) { + selectorUsed = selector; + break; + } + } + + // If no specific visualization element found, use the target element itself + if (!visualizationElement) { + visualizationElement = targetElement; + selectorUsed = 'targetElement'; + } + + // Use html2canvas to capture the visualization + const html2canvas = await import('html2canvas'); + const canvas = await html2canvas.default(visualizationElement, { + backgroundColor: '#ffffff', + scale: 1, + logging: false, + useCORS: true, + allowTaint: true, + height: visualizationElement.offsetHeight || 400, + width: visualizationElement.offsetWidth || 600, + }); + + return canvas.toDataURL('image/png'); + } catch (error) { + return null; + } +} + +export class AskAIAction implements ActionByType { + public readonly type = ACTION_ASK_AI; + public readonly id = ACTION_ASK_AI; + public order = 100; + + constructor(private core: CoreStart) {} + + public getDisplayName({ embeddable }: AskAIActionContext) { + if (!embeddable || isErrorEmbeddable(embeddable)) { + throw new IncompatibleActionError(); + } + return i18n.translate('visualizations.actions.askAI.displayName', { + defaultMessage: 'Ask AI', + }); + } + + public getIconType({ embeddable }: AskAIActionContext): EuiIconType { + if (!embeddable || isErrorEmbeddable(embeddable)) { + throw new IncompatibleActionError(); + } + return 'discuss' as EuiIconType; + } + + public async isCompatible({ embeddable }: AskAIActionContext) { + // Check if embeddable is valid and chat service is available + if (!embeddable || isErrorEmbeddable(embeddable)) { + return false; + } + + // Check if chat service is available + const chatService = this.core.chat; + if (!chatService || !chatService.isAvailable()) { + return false; + } + + // Only show for visualization embeddables + const embeddableType = embeddable.type; + const visualizationTypes = [ + 'visualization', + 'lens', + 'map', + 'vega', + 'timelion', + 'input_control_vis', + 'metrics', + 'tagcloud', + 'region_map', + 'tile_map', + 'histogram', + 'line', + 'area', + 'bar', + 'pie', + 'metric', + 'table', + 'gauge', + 'goal', + 'heatmap', + ]; + + return visualizationTypes.includes(embeddableType); + } + + public async execute({ embeddable }: AskAIActionContext) { + if (!embeddable || isErrorEmbeddable(embeddable)) { + throw new IncompatibleActionError(); + } + + const chatService = this.core.chat; + if (!chatService || !chatService.isAvailable()) { + this.core.notifications.toasts.addWarning({ + title: i18n.translate('visualizations.actions.askAI.chatUnavailable.title', { + defaultMessage: 'Chat service unavailable', + }), + text: i18n.translate('visualizations.actions.askAI.chatUnavailable.text', { + defaultMessage: 'The AI chat service is not available. Please check your configuration.', + }), + }); + return; + } + + try { + // Open chat window first + await chatService.openWindow(); + + // Wait a bit for the chat window to be fully rendered and ref to be established + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Show loading indicator + if (chatService.setCapturingImage) { + chatService.setCapturingImage(true); + } + + // Capture screenshot of the visualization + const screenshot = await captureVisualizationScreenshot(embeddable); + + // Hide loading indicator + if (chatService.setCapturingImage) { + chatService.setCapturingImage(false); + } + + if (!screenshot) { + this.core.notifications.toasts.addWarning({ + title: i18n.translate('visualizations.actions.askAI.screenshotFailed.title', { + defaultMessage: 'Screenshot capture failed', + }), + text: i18n.translate('visualizations.actions.askAI.screenshotFailed.text', { + defaultMessage: + 'Could not capture visualization screenshot. You can still ask questions in the chat.', + }), + }); + return; + } + + // Set the screenshot as pending image in the chat input + if (screenshot && chatService.setPendingImage) { + // Retry mechanism in case the chat window ref isn't ready yet + let retries = 0; + const maxRetries = 5; + const setImageWithRetry = async () => { + try { + chatService.setPendingImage!(screenshot); + } catch (error) { + if (retries < maxRetries) { + retries++; + await new Promise((resolve) => setTimeout(resolve, 200)); + await setImageWithRetry(); + } + } + }; + + await setImageWithRetry(); + } + } catch (error) { + // Make sure to hide loading indicator on error + if (chatService.setCapturingImage) { + chatService.setCapturingImage(false); + } + + this.core.notifications.toasts.addError(error, { + title: i18n.translate('visualizations.actions.askAI.error.title', { + defaultMessage: 'Failed to open chat', + }), + }); + } + } +} diff --git a/src/plugins/visualizations/public/actions/index.ts b/src/plugins/visualizations/public/actions/index.ts new file mode 100644 index 000000000000..aae7b169bbeb --- /dev/null +++ b/src/plugins/visualizations/public/actions/index.ts @@ -0,0 +1,6 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { AskAIAction, ACTION_ASK_AI, AskAIActionContext } from './ask_ai_action'; diff --git a/src/plugins/visualizations/public/plugin.ts b/src/plugins/visualizations/public/plugin.ts index 3912b722cc9c..9de0db9aef85 100644 --- a/src/plugins/visualizations/public/plugin.ts +++ b/src/plugins/visualizations/public/plugin.ts @@ -100,6 +100,14 @@ import { DashboardStart } from '../../dashboard/public'; import { createSavedAugmentVisLoader } from '../../vis_augmenter/public'; import { DocLinksStart } from '../../../core/public'; import { createNewVisActions } from './wizard/new_vis_actions'; +import { AskAIAction, ACTION_ASK_AI, AskAIActionContext } from './actions'; +import { CONTEXT_MENU_TRIGGER } from '../../embeddable/public'; + +declare module '../../ui_actions/public' { + export interface ActionContextMapping { + [ACTION_ASK_AI]: AskAIActionContext; + } +} /** * Interface for this plugin's returned setup/start contracts. @@ -205,6 +213,11 @@ export class VisualizationsPlugin savedObjects: core.savedObjects, }); + // Register Ask AI action + const askAIAction = new AskAIAction(core); + uiActions.registerAction(askAIAction); + uiActions.attachAction(CONTEXT_MENU_TRIGGER, askAIAction.id); + setDataStart(data); setSavedAugmentVisLoader(savedAugmentVisLoader); setI18n(core.i18n); diff --git a/yarn.lock b/yarn.lock index a16b0db3fbae..e7a563c04190 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10859,6 +10859,11 @@ bare-stream@^2.0.0: dependencies: streamx "^2.21.0" +base64-arraybuffer@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/base64-arraybuffer/-/base64-arraybuffer-1.0.2.tgz#1c37589a7c4b0746e34bd1feb951da2df01c1bdc" + integrity sha512-I3yl4r9QB5ZRY3XuJVEPfc2XhZO6YweFPI+UovAzn+8/hb3oJ6lnysaFcjVpkCPfVWFUDvoZ8kmVDP7WyRtYtQ== + base64-js@^1.0.2, base64-js@^1.3.1, base64-js@^1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" @@ -12582,6 +12587,13 @@ css-in-js-utils@^2.0.0: hyphenate-style-name "^1.0.2" isobject "^3.0.1" +css-line-break@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/css-line-break/-/css-line-break-2.1.0.tgz#bfef660dfa6f5397ea54116bb3cb4873edbc4fa0" + integrity sha512-FHcKFCZcAha3LwfVBhCQbW2nCNbkZXn7KVUJcsT5/P8YmfsVja0FMPJr0B903j/E69HUphKiV9iQArX8SDYA4w== + dependencies: + utrie "^1.0.2" + css-loader@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/css-loader/-/css-loader-3.6.0.tgz#2e4b2c7e6e2d27f8c8f28f61bffcd2e6c91ef645" @@ -16996,6 +17008,14 @@ html-webpack-plugin@^4.0.0: tapable "^1.1.3" util.promisify "1.0.0" +html2canvas@^1.4.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/html2canvas/-/html2canvas-1.4.1.tgz#7cef1888311b5011d507794a066041b14669a543" + integrity sha512-fPU6BHNpsyIhr8yyMpTLLxAbkaK8ArIBcmZIRiBLiDhjeqvXolaEmDGmELFuX9I4xDcaKKcJl+TKZLqruBbmWA== + dependencies: + css-line-break "^2.1.0" + text-segmentation "^1.0.3" + htmlparser2@^3.9.1: version "3.10.1" resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.10.1.tgz#bd679dc3f59897b6a34bb10749c855bb53a9392f" @@ -25542,7 +25562,7 @@ string-length@^4.0.1: char-regex "^1.0.2" strip-ansi "^6.0.0" -"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0": version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -25560,6 +25580,15 @@ string-width@^1.0.1: is-fullwidth-code-point "^1.0.0" strip-ansi "^3.0.0" +"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + string-width@^2.1.0, string-width@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/string-width/-/string-width-2.1.1.tgz#ab93f27a8dc13d28cac815c462143a6d9012ae9e" @@ -25727,7 +25756,7 @@ stringify-entities@^3.0.1: character-entities-legacy "^1.0.0" xtend "^4.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -25769,6 +25798,13 @@ strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" +strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + strip-ansi@^7.0.1: version "7.1.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" @@ -26358,6 +26394,13 @@ text-diff@^1.0.1: resolved "https://registry.yarnpkg.com/text-diff/-/text-diff-1.0.1.tgz#6c105905435e337857375c9d2f6ca63e453ff565" integrity sha1-bBBZBUNeM3hXN1ydL2ymPkU/9WU= +text-segmentation@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/text-segmentation/-/text-segmentation-1.0.3.tgz#52a388159efffe746b24a63ba311b6ac9f2d7943" + integrity sha512-iOiPUo/BGnZ6+54OsWxZidGCsdU8YbE4PSpdPinp7DeMtUJNJBoJ/ouUSTJjHkh1KntHaltHl/gDs2FC4i5+Nw== + dependencies: + utrie "^1.0.2" + text-table@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" @@ -27467,6 +27510,13 @@ utils-merge@1.0.1: resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" integrity sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA== +utrie@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/utrie/-/utrie-1.0.2.tgz#d42fe44de9bc0119c25de7f564a6ed1b2c87a645" + integrity sha512-1MLa5ouZiOmQzUbjbu9VmjLzn1QLXBhwpUa7kdLUQK+KQ5KA9I1vk5U4YHe/X2Ch7PYnJfWuWT+VbuxbGwljhw== + dependencies: + base64-arraybuffer "^1.0.2" + uuid-browser@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/uuid-browser/-/uuid-browser-3.1.0.tgz#0f05a40aef74f9e5951e20efbf44b11871e56410" @@ -28519,7 +28569,7 @@ workerpool@^6.5.1: resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.5.1.tgz#060f73b39d0caf97c6db64da004cd01b4c099544" integrity sha512-Fs4dNYcsdpYSAfVxhnl1L5zTksjvOJxtC5hzMNl+1t9B8hTJTdKDyZ5ju7ztgPy+ft9tBFXoOlDNiOT9WUXZlA== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -28545,6 +28595,15 @@ wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"