From 01ea13e6844f80ea392df58881be7f3dc87bfc85 Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Fri, 10 Oct 2025 14:51:45 +0200 Subject: [PATCH 1/7] Dynamic AI response status and related resources display --- .../SearchOrAskAi/AskAi/Chat.tsx | 7 +- .../SearchOrAskAi/AskAi/ChatMessage.tsx | 110 +++++++-------- .../SearchOrAskAi/AskAi/ChatMessageList.tsx | 2 +- .../SearchOrAskAi/AskAi/GeneratingStatus.tsx | 83 ++++++++++++ .../SearchOrAskAi/AskAi/RelatedResources.tsx | 127 ++++++++++++++++++ .../AskAi/AskAiUsecase.cs | 46 ++++++- 6 files changed, 317 insertions(+), 58 deletions(-) create mode 100644 src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/GeneratingStatus.tsx create mode 100644 src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/RelatedResources.tsx diff --git a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/Chat.tsx b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/Chat.tsx index b110742ec..f46cd55f6 100644 --- a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/Chat.tsx +++ b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/Chat.tsx @@ -15,7 +15,7 @@ import { } from '@elastic/eui' import { css } from '@emotion/react' import * as React from 'react' -import { useCallback, useEffect, useRef } from 'react' +import { useCallback, useEffect, useRef, useState } from 'react' const containerStyles = css` height: 100%; @@ -66,6 +66,7 @@ export const Chat = () => { const inputRef = useRef(null) const scrollRef = useRef(null) const lastMessageStatusRef = useRef(null) + const [inputValue, setInputValue] = useState('') const dynamicScrollableStyles = css` ${scrollableStyles} @@ -81,6 +82,7 @@ export const Chat = () => { if (inputRef.current) { inputRef.current.value = '' } + setInputValue('') // Scroll to bottom after new message setTimeout(() => scrollToBottom(scrollRef.current), 100) @@ -202,6 +204,7 @@ export const Chat = () => { inputRef={inputRef} fullWidth placeholder="Ask Elastic Docs AI Assistant" + onChange={(e) => setInputValue(e.target.value)} onKeyDown={(e) => { if (e.key === 'Enter') { handleSubmit(e.currentTarget.value) @@ -219,7 +222,7 @@ export const Chat = () => { `} color="primary" iconType="sortUp" - display="base" + display={inputValue.trim() ? 'fill' : 'base'} onClick={() => { if (inputRef.current) { handleSubmit(inputRef.current.value) diff --git a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/ChatMessage.tsx b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/ChatMessage.tsx index 856c7addd..5cd6f107c 100644 --- a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/ChatMessage.tsx +++ b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/ChatMessage.tsx @@ -1,8 +1,9 @@ import { initCopyButton } from '../../../copybutton' +import { GeneratingStatus } from './GeneratingStatus' +import { References } from './RelatedResources' import { ChatMessage as ChatMessageType } from './chat.store' import { LlmGatewayMessage } from './useLlmGateway' import { - EuiAvatar, EuiButtonIcon, EuiCallOut, EuiCopy, @@ -10,7 +11,6 @@ import { EuiFlexItem, EuiIcon, EuiLoadingElastic, - EuiLoadingSpinner, EuiPanel, EuiSpacer, EuiText, @@ -58,6 +58,24 @@ const getAccumulatedContent = (messages: LlmGatewayMessage[]) => { .join('') } +const splitContentAndReferences = ( + content: string +): { mainContent: string; referencesJson: string | null } => { + const delimiter = '--- references ---' + const delimiterIndex = content.indexOf(delimiter) + + if (delimiterIndex === -1) { + return { mainContent: content, referencesJson: null } + } + + const mainContent = content.substring(0, delimiterIndex).trim() + const referencesJson = content + .substring(delimiterIndex + delimiter.length) + .trim() + + return { mainContent, referencesJson } +} + const getMessageState = (message: ChatMessageType) => ({ isUser: message.type === 'user', isLoading: message.status === 'streaming', @@ -73,7 +91,7 @@ const ActionBar = ({ content: string onRetry?: () => void }) => ( - + - - - - - - {message.content} - - - + + {message.content} + + ) } @@ -176,10 +187,15 @@ export const ChatMessage = ({ const hasError = message.status === 'error' || !!error + const { mainContent, referencesJson } = useMemo( + () => splitContentAndReferences(content), + [content] + ) + const parsed = useMemo(() => { - const html = markedInstance.parse(content) as string + const html = markedInstance.parse(mainContent) as string return DOMPurify.sanitize(html) - }, [content]) + }, [mainContent]) const ref = React.useRef(null) @@ -239,16 +255,12 @@ export const ChatMessage = ({ {content && ( - // - // {content} - //
)} - {isLoading && ( - <> - {content && } - - - - - - - Generating... - - - - + {referencesJson && ( + )} + {content && isLoading && } + + {isComplete && content && ( <> diff --git a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/ChatMessageList.tsx b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/ChatMessageList.tsx index 1426b7a2d..2668431e0 100644 --- a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/ChatMessageList.tsx +++ b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/ChatMessageList.tsx @@ -21,7 +21,7 @@ export const ChatMessageList = ({ messages }: ChatMessageListProps) => { isLast={index === messages.length - 1} /> )} - {index < messages.length - 1 && } + {index < messages.length - 1 && } ))} diff --git a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/GeneratingStatus.tsx b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/GeneratingStatus.tsx new file mode 100644 index 000000000..5cb3096c0 --- /dev/null +++ b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/GeneratingStatus.tsx @@ -0,0 +1,83 @@ +/** @jsxImportSource @emotion/react */ +import { LlmGatewayMessage } from './useLlmGateway' +import { + EuiFlexGroup, + EuiFlexItem, + EuiLoadingSpinner, + EuiText, +} from '@elastic/eui' +import * as React from 'react' + +interface GeneratingStatusProps { + llmMessages: LlmGatewayMessage[] + isComplete?: boolean +} + +const getToolCallSearchQuery = ( + messages: LlmGatewayMessage[] +): string | null => { + const toolCallMessage = messages.find((m) => m.type === 'tool_call') + if (!toolCallMessage) return null + + try { + const toolCalls = toolCallMessage.data?.toolCalls + if (toolCalls && toolCalls.length > 0) { + const firstToolCall = toolCalls[0] + return firstToolCall.args?.searchQuery || null + } + } catch (e) { + console.error('Error extracting search query from tool call:', e) + } + + return null +} + +const hasContentStarted = (messages: LlmGatewayMessage[]): boolean => { + return messages.some((m) => m.type === 'ai_message_chunk' && m.data.content) +} + +const hasReachedReferences = (messages: LlmGatewayMessage[]): boolean => { + const accumulatedContent = messages + .filter((m) => m.type === 'ai_message_chunk') + .map((m) => m.data.content) + .join('') + return accumulatedContent.includes('--- references ---') +} + +export const GeneratingStatus = ({ + llmMessages, + isComplete = false, +}: GeneratingStatusProps) => { + const searchQuery = getToolCallSearchQuery(llmMessages) + const contentStarted = hasContentStarted(llmMessages) + const reachedReferences = hasReachedReferences(llmMessages) + + // If complete, don't show anything + if (isComplete) { + return null + } + + // Loading states + let statusText = 'Thinking' + + if (reachedReferences) { + statusText = 'Finding sources' + } else if (contentStarted) { + statusText = 'Generating' + } else if (searchQuery) { + statusText = `Searching for "${searchQuery}"` + } + + return ( + + + + + + + {statusText}... + + + + ) +} diff --git a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/RelatedResources.tsx b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/RelatedResources.tsx new file mode 100644 index 000000000..b7bfacefb --- /dev/null +++ b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/RelatedResources.tsx @@ -0,0 +1,127 @@ +/** @jsxImportSource @emotion/react */ +import { + EuiFlexGroup, + EuiFlexItem, + EuiText, + EuiLink, + EuiSpacer, + EuiIcon, + EuiPanel, + useEuiTheme, +} from '@elastic/eui' +import { css } from '@emotion/react' +import * as React from 'react' + +interface Reference { + url: string + title: string + description: string +} + +interface ReferencesProps { + referencesJson: string +} + +const parseReferences = (jsonString: string): Reference[] => { + try { + // Remove markdown code fences if present + let cleanedJson = jsonString.trim() + + // Check for ```json or ``` at the start + if (cleanedJson.startsWith('```json')) { + cleanedJson = cleanedJson.substring(7) // Remove ```json + } else if (cleanedJson.startsWith('```')) { + cleanedJson = cleanedJson.substring(3) // Remove ``` + } + + // Remove closing ``` at the end + if (cleanedJson.endsWith('```')) { + cleanedJson = cleanedJson.substring(0, cleanedJson.length - 3) + } + + cleanedJson = cleanedJson.trim() + + const parsed = JSON.parse(cleanedJson) + if (Array.isArray(parsed)) { + return parsed + } + return [] + } catch (e) { + console.error('Failed to parse references JSON:', e) + return [] + } +} + +export const References = ({ referencesJson }: ReferencesProps) => { + const references = parseReferences(referencesJson) + const { euiTheme } = useEuiTheme() + + if (references.length === 0) { + return null + } + + return ( + <> + + +
+ + Related resources: + + + + {references.map((ref, index) => ( + + + + + + +
+ + {ref.title} + + + {ref.description} + +
+ + + + ))} + +
+ + + ) +} diff --git a/src/api/Elastic.Documentation.Api.Core/AskAi/AskAiUsecase.cs b/src/api/Elastic.Documentation.Api.Core/AskAi/AskAiUsecase.cs index c39b23c38..4c2bd7243 100644 --- a/src/api/Elastic.Documentation.Api.Core/AskAi/AskAiUsecase.cs +++ b/src/api/Elastic.Documentation.Api.Core/AskAi/AskAiUsecase.cs @@ -34,6 +34,50 @@ public record AskAiRequest(string Message, string? ThreadId) - Do not mention that you are a language model or AI. - Do not provide answers based on your general knowledge. - - Do not ask the user for clarification. + + ## Formatting Guidelines: + - Use Markdown for formatting your response. + - Use headings, bullet points, and numbered lists to organize information clearly. + - Use sentence case for headings. + + ## Sources and References Extraction *IMPORTANT*: + - Do *NOT* add a heading for the sources section. + - When you provide an answer, *ALWAYS* include a refernces at the end of your response. + - List all relevant document titles or sections that you referenced to formulate your answer. + - Only use the documents provided to you; do not reference any external sources. + - If no relevant documents were used, state "No sources available." + - Use this schema: + { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "List of Documentation Resources", + "description": "A list of objects, each representing a documentation resource with a URL, title, and description.", + "type": "array", + "items": { + "type": "object", + "properties": { + "url": { + "description": "The URL of the resource.", + "type": "string", + "format": "uri" + }, + "title": { + "description": "The title of the resource.", + "type": "string" + }, + "description": { + "description": "A brief description of the resource.", + "type": "string" + } + }, + "required": [ + "url", + "title", + "description" + ] + } + } + - Ensure that the URLs you provide are directly relevant to the user's question and the content of the documents. + - Add a delimiter "--- references ---" before the sources section + """; } From 7109c6db094299347ae94b6115cd8038474c1e0c Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Fri, 10 Oct 2025 14:58:57 +0200 Subject: [PATCH 2/7] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../web-components/SearchOrAskAi/AskAi/RelatedResources.tsx | 2 +- src/api/Elastic.Documentation.Api.Core/AskAi/AskAiUsecase.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/RelatedResources.tsx b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/RelatedResources.tsx index b7bfacefb..fe471ffa4 100644 --- a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/RelatedResources.tsx +++ b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/RelatedResources.tsx @@ -47,7 +47,7 @@ const parseReferences = (jsonString: string): Reference[] => { } return [] } catch (e) { - console.error('Failed to parse references JSON:', e) + console.error('Failed to parse references JSON:', e, 'Input:', jsonString) return [] } } diff --git a/src/api/Elastic.Documentation.Api.Core/AskAi/AskAiUsecase.cs b/src/api/Elastic.Documentation.Api.Core/AskAi/AskAiUsecase.cs index 4c2bd7243..5c7d8ddca 100644 --- a/src/api/Elastic.Documentation.Api.Core/AskAi/AskAiUsecase.cs +++ b/src/api/Elastic.Documentation.Api.Core/AskAi/AskAiUsecase.cs @@ -42,7 +42,7 @@ public record AskAiRequest(string Message, string? ThreadId) ## Sources and References Extraction *IMPORTANT*: - Do *NOT* add a heading for the sources section. - - When you provide an answer, *ALWAYS* include a refernces at the end of your response. + - When you provide an answer, *ALWAYS* include a references at the end of your response. - List all relevant document titles or sections that you referenced to formulate your answer. - Only use the documents provided to you; do not reference any external sources. - If no relevant documents were used, state "No sources available." From 20932240fcf2796ff6f3dd9a12a029ac0214921a Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Fri, 10 Oct 2025 15:24:00 +0200 Subject: [PATCH 3/7] Run prettier --- .../SearchOrAskAi/AskAi/RelatedResources.tsx | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/RelatedResources.tsx b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/RelatedResources.tsx index fe471ffa4..7b62f824a 100644 --- a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/RelatedResources.tsx +++ b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/RelatedResources.tsx @@ -47,7 +47,12 @@ const parseReferences = (jsonString: string): Reference[] => { } return [] } catch (e) { - console.error('Failed to parse references JSON:', e, 'Input:', jsonString) + console.error( + 'Failed to parse references JSON:', + e, + 'Input:', + jsonString + ) return [] } } From 62f372b6deb7c90a83027ccb54d8eb103d46f4d3 Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Sat, 11 Oct 2025 01:00:13 +0200 Subject: [PATCH 4/7] Refactor --- .../SearchOrAskAi/AskAi/ChatMessage.tsx | 82 +++++++++++++++++-- .../SearchOrAskAi/AskAi/GeneratingStatus.tsx | 60 +------------- .../SearchOrAskAi/AskAi/RelatedResources.tsx | 8 +- .../SearchOrAskAi/Search/SearchResults.tsx | 16 +--- .../AskAi/AskAiUsecase.cs | 11 ++- 5 files changed, 91 insertions(+), 86 deletions(-) diff --git a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/ChatMessage.tsx b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/ChatMessage.tsx index 5cd6f107c..fe083c84a 100644 --- a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/ChatMessage.tsx +++ b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/ChatMessage.tsx @@ -83,6 +83,59 @@ const getMessageState = (message: ChatMessageType) => ({ hasError: message.status === 'error', }) +// Helper functions for computing AI status +const getToolCallSearchQuery = ( + messages: LlmGatewayMessage[] +): string | null => { + const toolCallMessage = messages.find((m) => m.type === 'tool_call') + if (!toolCallMessage) return null + + try { + const toolCalls = toolCallMessage.data?.toolCalls + if (toolCalls && toolCalls.length > 0) { + const firstToolCall = toolCalls[0] + return firstToolCall.args?.searchQuery || null + } + } catch (e) { + console.error('Error extracting search query from tool call:', e) + } + + return null +} + +const hasContentStarted = (messages: LlmGatewayMessage[]): boolean => { + return messages.some((m) => m.type === 'ai_message_chunk' && m.data.content) +} + +const hasReachedReferences = (messages: LlmGatewayMessage[]): boolean => { + const accumulatedContent = messages + .filter((m) => m.type === 'ai_message_chunk') + .map((m) => m.data.content) + .join('') + return accumulatedContent.includes('--- references ---') +} + +const computeAiStatus = ( + llmMessages: LlmGatewayMessage[], + isComplete: boolean +): string | null => { + if (isComplete) return null + + const searchQuery = getToolCallSearchQuery(llmMessages) + const contentStarted = hasContentStarted(llmMessages) + const reachedReferences = hasReachedReferences(llmMessages) + + if (reachedReferences) { + return 'Gathering resources' + } else if (contentStarted) { + return 'Generating' + } else if (searchQuery) { + return `Searching for "${searchQuery}"` + } + + return 'Thinking' +} + // Action bar for complete AI messages const ActionBar = ({ content, @@ -187,16 +240,30 @@ export const ChatMessage = ({ const hasError = message.status === 'error' || !!error - const { mainContent, referencesJson } = useMemo( - () => splitContentAndReferences(content), - [content] - ) + // Only split content and references when complete for better performance + const { mainContent, referencesJson } = useMemo(() => { + if (isComplete) { + return splitContentAndReferences(content) + } + // During streaming, strip out unparsed references but don't parse them yet + const delimiter = '--- references ---' + const delimiterIndex = content.indexOf(delimiter) + if (delimiterIndex !== -1) { + return { mainContent: content.substring(0, delimiterIndex).trim(), referencesJson: null } + } + return { mainContent: content, referencesJson: null } + }, [content, isComplete]) const parsed = useMemo(() => { const html = markedInstance.parse(mainContent) as string return DOMPurify.sanitize(html) }, [mainContent]) + const aiStatus = useMemo( + () => computeAiStatus(llmMessages, isComplete), + [llmMessages, isComplete] + ) + const ref = React.useRef(null) useEffect(() => { @@ -279,15 +346,12 @@ export const ChatMessage = ({ )} {content && isLoading && } - + {isComplete && content && ( <> - + )} diff --git a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/GeneratingStatus.tsx b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/GeneratingStatus.tsx index 5cb3096c0..da543d91b 100644 --- a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/GeneratingStatus.tsx +++ b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/GeneratingStatus.tsx @@ -1,5 +1,4 @@ /** @jsxImportSource @emotion/react */ -import { LlmGatewayMessage } from './useLlmGateway' import { EuiFlexGroup, EuiFlexItem, @@ -9,65 +8,14 @@ import { import * as React from 'react' interface GeneratingStatusProps { - llmMessages: LlmGatewayMessage[] - isComplete?: boolean + status: string | null } -const getToolCallSearchQuery = ( - messages: LlmGatewayMessage[] -): string | null => { - const toolCallMessage = messages.find((m) => m.type === 'tool_call') - if (!toolCallMessage) return null - - try { - const toolCalls = toolCallMessage.data?.toolCalls - if (toolCalls && toolCalls.length > 0) { - const firstToolCall = toolCalls[0] - return firstToolCall.args?.searchQuery || null - } - } catch (e) { - console.error('Error extracting search query from tool call:', e) - } - - return null -} - -const hasContentStarted = (messages: LlmGatewayMessage[]): boolean => { - return messages.some((m) => m.type === 'ai_message_chunk' && m.data.content) -} - -const hasReachedReferences = (messages: LlmGatewayMessage[]): boolean => { - const accumulatedContent = messages - .filter((m) => m.type === 'ai_message_chunk') - .map((m) => m.data.content) - .join('') - return accumulatedContent.includes('--- references ---') -} - -export const GeneratingStatus = ({ - llmMessages, - isComplete = false, -}: GeneratingStatusProps) => { - const searchQuery = getToolCallSearchQuery(llmMessages) - const contentStarted = hasContentStarted(llmMessages) - const reachedReferences = hasReachedReferences(llmMessages) - - // If complete, don't show anything - if (isComplete) { +export const GeneratingStatus = ({ status }: GeneratingStatusProps) => { + if (!status) { return null } - // Loading states - let statusText = 'Thinking' - - if (reachedReferences) { - statusText = 'Finding sources' - } else if (contentStarted) { - statusText = 'Generating' - } else if (searchQuery) { - statusText = `Searching for "${searchQuery}"` - } - return ( @@ -75,7 +23,7 @@ export const GeneratingStatus = ({ - {statusText}... + {status}... diff --git a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/RelatedResources.tsx b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/RelatedResources.tsx index 7b62f824a..efb9ec8f4 100644 --- a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/RelatedResources.tsx +++ b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/RelatedResources.tsx @@ -47,12 +47,6 @@ const parseReferences = (jsonString: string): Reference[] => { } return [] } catch (e) { - console.error( - 'Failed to parse references JSON:', - e, - 'Input:', - jsonString - ) return [] } } @@ -67,7 +61,7 @@ export const References = ({ referencesJson }: ReferencesProps) => { return ( <> - + - + {parent.title} diff --git a/src/api/Elastic.Documentation.Api.Core/AskAi/AskAiUsecase.cs b/src/api/Elastic.Documentation.Api.Core/AskAi/AskAiUsecase.cs index 5c7d8ddca..fa35355fa 100644 --- a/src/api/Elastic.Documentation.Api.Core/AskAi/AskAiUsecase.cs +++ b/src/api/Elastic.Documentation.Api.Core/AskAi/AskAiUsecase.cs @@ -34,6 +34,8 @@ public record AskAiRequest(string Message, string? ThreadId) - Do not mention that you are a language model or AI. - Do not provide answers based on your general knowledge. + - Do not add a heading for the references section. + - Do not include any preamble or explanation before the sources section. ## Formatting Guidelines: - Use Markdown for formatting your response. @@ -41,12 +43,15 @@ public record AskAiRequest(string Message, string? ThreadId) - Use sentence case for headings. ## Sources and References Extraction *IMPORTANT*: - - Do *NOT* add a heading for the sources section. + - When you provide an answer, *ALWAYS* include a references at the end of your response. - List all relevant document titles or sections that you referenced to formulate your answer. + - Also add the links of the documents you used in your answer. - Only use the documents provided to you; do not reference any external sources. - - If no relevant documents were used, state "No sources available." + - If no relevant documents were used return an empty list. + - The JSON is hidden from the user so exclude any preamble or explanation about it. - Use this schema: + ``` { "$schema": "http://json-schema.org/draft-07/schema#", "title": "List of Documentation Resources", @@ -66,6 +71,7 @@ public record AskAiRequest(string Message, string? ThreadId) }, "description": { "description": "A brief description of the resource.", + "maxLength": 150, "type": "string" } }, @@ -76,6 +82,7 @@ public record AskAiRequest(string Message, string? ThreadId) ] } } + ``` - Ensure that the URLs you provide are directly relevant to the user's question and the content of the documents. - Add a delimiter "--- references ---" before the sources section From 55fa36aeb30eb14347e69926ebf23e20a4d3ec72 Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Sat, 11 Oct 2025 01:42:45 +0200 Subject: [PATCH 5/7] Use html comment delimiter To increase the chance that the json will not be rendered on the UI --- .../SearchOrAskAi/AskAi/ChatMessage.tsx | 32 +++++++++++++------ .../SearchOrAskAi/AskAi/RelatedResources.tsx | 1 + .../SearchOrAskAi/Search/SearchResults.tsx | 11 ++++--- .../AskAi/AskAiUsecase.cs | 9 +++++- 4 files changed, 37 insertions(+), 16 deletions(-) diff --git a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/ChatMessage.tsx b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/ChatMessage.tsx index fe083c84a..0e8bb6f08 100644 --- a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/ChatMessage.tsx +++ b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/ChatMessage.tsx @@ -61,16 +61,22 @@ const getAccumulatedContent = (messages: LlmGatewayMessage[]) => { const splitContentAndReferences = ( content: string ): { mainContent: string; referencesJson: string | null } => { - const delimiter = '--- references ---' - const delimiterIndex = content.indexOf(delimiter) + const startDelimiter = '' - if (delimiterIndex === -1) { + const startIndex = content.indexOf(startDelimiter) + if (startIndex === -1) { return { mainContent: content, referencesJson: null } } - const mainContent = content.substring(0, delimiterIndex).trim() + const endIndex = content.indexOf(endDelimiter, startIndex) + if (endIndex === -1) { + return { mainContent: content, referencesJson: null } + } + + const mainContent = content.substring(0, startIndex).trim() const referencesJson = content - .substring(delimiterIndex + delimiter.length) + .substring(startIndex + startDelimiter.length, endIndex) .trim() return { mainContent, referencesJson } @@ -112,7 +118,7 @@ const hasReachedReferences = (messages: LlmGatewayMessage[]): boolean => { .filter((m) => m.type === 'ai_message_chunk') .map((m) => m.data.content) .join('') - return accumulatedContent.includes('--- references ---') + return accumulatedContent.includes(' + ``` """; } From 8bed643768fc26b71c27f1c2ca93452600fdae82 Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Sat, 11 Oct 2025 01:59:50 +0200 Subject: [PATCH 6/7] Simplify prompt --- .../AskAi/AskAiUsecase.cs | 143 +++++++++--------- 1 file changed, 70 insertions(+), 73 deletions(-) diff --git a/src/api/Elastic.Documentation.Api.Core/AskAi/AskAiUsecase.cs b/src/api/Elastic.Documentation.Api.Core/AskAi/AskAiUsecase.cs index 0bf07a880..1d8579c10 100644 --- a/src/api/Elastic.Documentation.Api.Core/AskAi/AskAiUsecase.cs +++ b/src/api/Elastic.Documentation.Api.Core/AskAi/AskAiUsecase.cs @@ -19,79 +19,76 @@ public record AskAiRequest(string Message, string? ThreadId) { public static string SystemPrompt => """ - Role: You are a specialized AI assistant designed to answer user questions exclusively from a set of provided documentation. Your primary purpose is to retrieve, synthesize, and present information directly from these documents. - - ## Core Directives: - - - Source of Truth: Your only source of information is the document content provided to you for each user query. You must not use any pre-trained knowledge or external information. - - Answering Style: Answer the user's question directly and comprehensively. As the user cannot ask follow-up questions, your response must be a complete, self-contained answer to their query. Do not start with phrases like "Based on the documents..."—simply provide the answer. - - Handling Unknowns: If the information required to answer the question is not present in the provided documents, you must explicitly state that the answer cannot be found. Do not attempt to guess, infer, or provide a general response. - - Helpful Fallback: If you cannot find a direct answer, you may suggest and link to a few related or similar topics that are present in the documentation. This provides value even when a direct answer is unavailable. - - Output Format: Your final response should be a single, coherent block of text. - - Short and Concise: Keep your answers as brief as possible while still being complete and informative. For more more details refer to the documentation with links. - - ## Negative Constraints: - - - Do not mention that you are a language model or AI. - - Do not provide answers based on your general knowledge. - - Do not add a heading for the references section. - - Do not include any preamble or explanation before the sources section. - - ## Formatting Guidelines: - - Use Markdown for formatting your response. - - Use headings, bullet points, and numbered lists to organize information clearly. - - Use sentence case for headings. - - ## Sources and References Extraction *IMPORTANT*: - - - When you provide an answer, *ALWAYS* include a references at the end of your response. - - List all relevant document titles or sections that you referenced to formulate your answer. - - Also add the links of the documents you used in your answer. - - Only use the documents provided to you; do not reference any external sources. - - If no relevant documents were used return an empty list. - - The JSON is hidden from the user so exclude any preamble or explanation about it. - - Use this schema: - ``` - { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "List of Documentation Resources", - "description": "A list of objects, each representing a documentation resource with a URL, title, and description.", - "type": "array", - "items": { - "type": "object", - "properties": { - "url": { - "description": "The URL of the resource.", - "type": "string", - "format": "uri" - }, - "title": { - "description": "The title of the resource.", - "type": "string" - }, - "description": { - "description": "A brief description of the resource.", - "maxLength": 150, - "type": "string" - } - }, - "required": [ - "url", - "title", - "description" - ] - } - } - ``` - - Ensure that the URLs you provide are directly relevant to the user's question and the content of the documents. - - Add a multi-line delimiter before the sources section using this exact format: - ``` - - ``` + You are an expert documentation assistant. Your primary task is to answer user questions using **only** the provided documentation. + ## Task Overview + Synthesize information from the provided text to give a direct, comprehensive, and self-contained answer to the user's query. + + --- + + ## Critical Rules + 1. **Strictly Adhere to Provided Sources:** Your ONLY source of information is the document content provided with by your RAG search. **DO NOT** use any of your pre-trained knowledge or external information. + 2. **Handle Unanswerable Questions:** If the answer is not in the documents, you **MUST** state this explicitly (e.g., "The answer to your question could not be found in the provided documentation."). Do not infer, guess, or provide a general knowledge answer. As a helpful fallback, you may suggest a few related topics that *are* present in the documentation. + 3. **Be Direct and Anonymous:** Answer the question directly without any preamble like "Based on the documents..." or "In the provided text...". **DO NOT** mention that you are an AI or language model. + + --- + + ## Response Formatting + + ### 1. User-Visible Answer + * The final response must be a single, coherent block of text. + * Format your answer using Markdown (headings, bullet points, etc.) for clarity. + * Use sentence case for all headings. + * Keep your answers concise yet complete. Answer the user's question fully, but link to the source documents for more extensive details. + + ### 2. Hidden Source References (*Crucial*) + * At the end of your response, you **MUST** **ALWAYS** provide a list of all documents you used to formulate the answer. + * Also include links that you used in your answer. + * This list must be a JSON array wrapped inside a specific multi-line comment delimiter. + * DO NOT add any headings, preamble, or explanations around the reference block. The JSON must be invisible to the end-user. + + **Delimiter and JSON Schema:** + + Use this exact format. The JSON array goes inside the comment block like the example below: + + ```markdown + + ``` + + **JSON Schema Definition:** + ```json + { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "List of Documentation Resources", + "description": "A list of objects, each representing a documentation resource with a URL, title, and description.", + "type": "array", + "items": { + "type": "object", + "properties": { + "url": { + "description": "The URL of the resource.", + "type": "string", + "format": "uri" + }, + "title": { + "description": "The title of the resource.", + "type": "string" + }, + "description": { + "description": "A brief description of the resource.", + "type": "string" + } + }, + "required": [ + "url", + "title", + "description" + ] + } + } """; } From 39cafb6f8fd10d91068725f86103e678c483063e Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Sat, 11 Oct 2025 02:37:59 +0200 Subject: [PATCH 7/7] Update prompt --- src/api/Elastic.Documentation.Api.Core/AskAi/AskAiUsecase.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/api/Elastic.Documentation.Api.Core/AskAi/AskAiUsecase.cs b/src/api/Elastic.Documentation.Api.Core/AskAi/AskAiUsecase.cs index 1d8579c10..01366b9d0 100644 --- a/src/api/Elastic.Documentation.Api.Core/AskAi/AskAiUsecase.cs +++ b/src/api/Elastic.Documentation.Api.Core/AskAi/AskAiUsecase.cs @@ -39,6 +39,7 @@ public record AskAiRequest(string Message, string? ThreadId) * The final response must be a single, coherent block of text. * Format your answer using Markdown (headings, bullet points, etc.) for clarity. * Use sentence case for all headings. + * Do not use `---` or any other section dividers in your answer. * Keep your answers concise yet complete. Answer the user's question fully, but link to the source documents for more extensive details. ### 2. Hidden Source References (*Crucial*)