Add highlighted body text to search results (#2008)

* Use search highlighting

* Revert Markdown.cs

* Remove DisableDirectStreaming again

* Run prettier

* Remove unused import

* Apply suggestions from code review

Co-authored-by: Copilot <[email protected]>

---------

Co-authored-by: Copilot <[email protected]>

Author: reakaleek

src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/Search/SearchResults.tsx

@@ -2,7 +2,6 @@ import { useSearchTerm } from '../search.store'
 import { SearchResultItem, useSearchQuery } from './useSearchQuery'
 import {
     useEuiFontSize,
-    EuiHighlight,
     EuiLink,
     EuiLoadingSpinner,
     EuiSpacer,
@@ -14,8 +13,8 @@ import {
 } from '@elastic/eui'
 import { css } from '@emotion/react'
 import { useDebounce } from '@uidotdev/usehooks'
-import * as React from 'react'
-import { useEffect, useMemo, useState } from 'react'
+import DOMPurify from 'dompurify'
+import { useEffect, useMemo, useState, memo } from 'react'
 
 export const SearchResults = () => {
     const searchTerm = useSearchTerm()
@@ -103,34 +102,23 @@ interface SearchResultListItemProps {
 
 function SearchResultListItem({ item: result }: SearchResultListItemProps) {
     const { euiTheme } = useEuiTheme()
-    const searchTerm = useSearchTerm()
-    const highlightSearchTerms = useMemo(
-        () =>
-            searchTerm
-                .toLowerCase()
-                .split(' ')
-                .filter((i) => i.length > 1),
-        [searchTerm]
-    )
-
-    if (highlightSearchTerms.includes('esql')) {
-        highlightSearchTerms.push('es|ql')
-    }
-
-    if (highlightSearchTerms.includes('dotnet')) {
-        highlightSearchTerms.push('.net')
-    }
+    const titleFontSize = useEuiFontSize('m')
     return (
-        <li>
+        <li
+            css={css`
+                :not(:first-child) {
+                    border-top: 1px dotted ${euiTheme.colors.borderBasePrimary};
+                }
+            `}
+        >
             <div
                 tabIndex={0}
                 css={css`
                             display: flex; 
                             align-items: flex-start;
                             gap: ${euiTheme.size.s};
                             padding-inline: ${euiTheme.size.s};
-                            padding-block: ${euiTheme.size.xs};
-                            border-radius: ${euiTheme.border.radius.small};
+                            padding-block: ${euiTheme.size.m};
                             :hover {
                                 background-color: ${euiTheme.colors.backgroundTransparentSubdued};
                         `}
@@ -148,41 +136,55 @@ function SearchResultListItem({ item: result }: SearchResultListItemProps) {
                         text-align: left;
                     `}
                 >
-                    <EuiLink
-                        tabIndex={-1}
-                        href={result.url}
+                    <Breadcrumbs parents={result.parents} />
+                    <div
                         css={css`
-                            .euiMark {
-                                background-color: ${euiTheme.colors
-                                    .backgroundLightWarning};
-                                font-weight: inherit;
-                            }
+                            padding-block: ${euiTheme.size.xs};
+                            font-size: ${titleFontSize.fontSize};
                         `}
                     >
-                        <EuiHighlight
-                            search={highlightSearchTerms}
-                            highlightAll={true}
+                        <EuiLink tabIndex={-1} href={result.url}>
+                            <span>{result.title}</span>
+                        </EuiLink>
+                    </div>
+
+                    <EuiText size="xs">
+                        <div
+                            css={css`
+                                font-family: ${euiTheme.font.family};
+                                position: relative;
+
+                                /* 2 lines with ellipsis */
+                                display: -webkit-box;
+                                -webkit-line-clamp: 2;
+                                -webkit-box-orient: vertical;
+                                overflow: hidden;
+
+                                width: 90%;
+
+                                mark {
+                                    background-color: transparent;
+                                    font-weight: ${euiTheme.font.weight.bold};
+                                    color: ${euiTheme.colors.ink};
+                                }
+                            `}
                         >
-                            {result.title}
-                        </EuiHighlight>
-                    </EuiLink>
-                    <Breadcrumbs
-                        parents={result.parents}
-                        highlightSearchTerms={highlightSearchTerms}
-                    />
+                            {result.highlightedBody ? (
+                                <SanitizedHtmlContent
+                                    htmlContent={result.highlightedBody}
+                                />
+                            ) : (
+                                <span>{result.description}</span>
+                            )}
+                        </div>
+                    </EuiText>
                 </div>
             </div>
         </li>
     )
 }
 
-function Breadcrumbs({
-    parents,
-    highlightSearchTerms,
-}: {
-    parents: SearchResultItem['parents']
-    highlightSearchTerms: string[]
-}) {
+function Breadcrumbs({ parents }: { parents: SearchResultItem['parents'] }) {
     const { euiTheme } = useEuiTheme()
     const { fontSize: smallFontsize } = useEuiFontSize('xs')
     return (
@@ -224,16 +226,40 @@ function Breadcrumbs({
                                     }
                                 `}
                             >
-                                <EuiHighlight
-                                    search={highlightSearchTerms}
-                                    highlightAll={true}
-                                >
-                                    {parent.title}
-                                </EuiHighlight>
+                                {parent.title}
                             </EuiText>
                         </EuiLink>
                     </li>
                 ))}
         </ul>
     )
 }
+
+const SanitizedHtmlContent = memo(
+    ({ htmlContent }: { htmlContent: string }) => {
+        const processed = useMemo(() => {
+            if (!htmlContent) return ''
+
+            const sanitized = DOMPurify.sanitize(htmlContent, {
+                ALLOWED_TAGS: ['mark'],
+                ALLOWED_ATTR: [],
+                KEEP_CONTENT: true,
+            })
+
+            // Check if text starts mid-sentence (lowercase first letter)
+            const temp = document.createElement('div')
+            temp.innerHTML = sanitized
+            const text = temp.textContent || ''
+            const firstChar = text.trim()[0]
+
+            // Add leading ellipsis if starts with lowercase
+            if (firstChar && /[a-z]/.test(firstChar)) {
+                return '… ' + sanitized
+            }
+
+            return sanitized
+        }, [htmlContent])
+
+        return <div dangerouslySetInnerHTML={{ __html: processed }} />
+    }
+)

src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/Search/useSearchQuery.ts

@@ -13,6 +13,7 @@ const SearchResultItem = z.object({
     description: z.string(),
     score: z.number(),
     parents: z.array(SearchResultItemParent),
+    highlightedBody: z.string().nullish(),
 })
 
 export type SearchResultItem = z.infer<typeof SearchResultItem>

src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/SearchOrAskAiModal.tsx

@@ -48,7 +48,7 @@ export const SearchOrAskAiModal = () => {
                 css={css`
                     flex-grow: 1;
                     overflow-y: scroll;
-                    max-height: 80vh;
+                    max-height: 70vh;
                     ${useEuiOverflowScroll('y')}
                 `}
             >

src/Elastic.Documentation.Site/package-lock.json

@@ -89,6 +89,7 @@
     "@tanstack/react-query": "^5.87.4",
     "@uidotdev/usehooks": "2.4.1",
     "clipboard": "2.0.11",
+    "dompurify": "3.2.7",
     "highlight.js": "11.11.1",
     "htmx-ext-head-support": "2.0.4",
     "htmx-ext-preload": "2.1.1",

src/Elastic.Documentation.Site/package.json

@@ -45,6 +45,10 @@ public record DocumentationDocument
 	[JsonPropertyName("body")]
 	public string? Body { get; set; }
 
+	// Stripped body is the body with markdown removed, suitable for search indexing
+	[JsonPropertyName("stripped_body")]
+	public string? StrippedBody { get; set; }
+
 	[JsonPropertyName("url_segment_count")]
 	public int? UrlSegmentCount { get; set; }
 

src/Elastic.Documentation/Search/DocumentationDocument.cs

@@ -12,6 +12,7 @@
 using Elastic.Ingest.Elasticsearch;
 using Elastic.Ingest.Elasticsearch.Catalog;
 using Elastic.Ingest.Elasticsearch.Semantic;
+using Elastic.Markdown.Helpers;
 using Elastic.Markdown.IO;
 using Elastic.Transport;
 using Elastic.Transport.Products.Elasticsearch;
@@ -90,13 +91,24 @@ protected static string CreateMappingSetting() =>
 		          "synonyms_filter"
 		        ]
 		      },
+		      "highlight_analyzer": {
+		        "tokenizer": "standard",
+		        "filter": [
+		          "lowercase",
+		          "english_stop"
+		        ]
+		      },
 		      "hierarchy_analyzer": { "tokenizer": "path_tokenizer" }
 		    },
 		    "filter": {
 		      "synonyms_filter": {
 		        "type": "synonym",
 		        "synonyms_set": "docs",
 		        "updateable": true
+		      },
+		      "english_stop": {
+		        "type": "stop",
+		        "stopwords": "_english_"
 		      }
 		    },
 		    "tokenizer": {
@@ -136,6 +148,11 @@ protected static string CreateMapping(string? inferenceId) =>
 		      },
 		      "body": {
 		        "type": "text"
+		      },
+		      "stripped_body": {
+		        "type": "text",
+		        "search_analyzer": "highlight_analyzer",
+		        "term_vector": "with_positions_offsets"
 		      }
 		      {{(!string.IsNullOrWhiteSpace(inferenceId) ? AbstractInferenceMapping(inferenceId) : AbstractMapping())}}
 		    }
@@ -277,11 +294,16 @@ public async ValueTask<bool> ExportAsync(MarkdownExportFileContext fileContext,
 
 		IPositionalNavigation navigation = fileContext.DocumentationSet;
 
-		//use LLM text if it was already provided (because we run with both llm and elasticsearch output)
-		var body = fileContext.LLMText ??= LlmMarkdownExporter.ConvertToLlmMarkdown(fileContext.Document, fileContext.BuildContext);
+		// Remove the first h1 because we already have the title
+		// and we don't want it to appear in the body
+		var h1 = fileContext.Document.Descendants<HeadingBlock>().FirstOrDefault(h => h.Level == 1);
+		if (h1 is not null)
+			_ = fileContext.Document.Remove(h1);
+
+		var body = LlmMarkdownExporter.ConvertToLlmMarkdown(fileContext.Document, fileContext.BuildContext);
 
 		var headings = fileContext.Document.Descendants<HeadingBlock>()
-			.Select(h => h.GetData("header") as string ?? string.Empty)
+			.Select(h => h.GetData("header") as string ?? string.Empty) // TODO: Confirm that 'header' data is correctly set for all HeadingBlock instances and that this extraction is reliable.
 			.Where(text => !string.IsNullOrEmpty(text))
 			.ToArray();
 
@@ -295,6 +317,7 @@ public async ValueTask<bool> ExportAsync(MarkdownExportFileContext fileContext,
 			Hash = ShortId.Create(url, body),
 			Title = file.Title,
 			Body = body,
+			StrippedBody = body.StripMarkdown(),
 			Description = fileContext.SourceFile.YamlFrontMatter?.Description,
 			Abstract = @abstract,
 			Applies = fileContext.SourceFile.YamlFrontMatter?.AppliesTo,
@@ -318,4 +341,3 @@ public async ValueTask<bool> FinishExportAsync(IDirectoryInfo outputFolder, Canc
 		return await _channel.RefreshAsync(ctx);
 	}
 }
-

src/Elastic.Markdown/Exporters/ElasticsearchMarkdownExporter.cs

@@ -17,7 +17,6 @@ public record MarkdownExportFileContext
 	public required MarkdownDocument Document { get; init; }
 	public required MarkdownFile SourceFile { get; init; }
 	public required IFileInfo DefaultOutputFile { get; init; }
-	public string? LLMText { get; set; }
 	public required DocumentationSet DocumentationSet { get; init; }
 }
 

src/Elastic.Markdown/Exporters/IMarkdownExporter.cs

@@ -19,26 +19,26 @@ public class LlmMarkdownExporter : IMarkdownExporter
 {
 	private const string LlmsTxtTemplate = """
 		# Elastic Documentation
-		
+
 		> Elastic provides an open source search, analytics, and AI platform, and out-of-the-box solutions for observability and security. The Search AI platform combines the power of search and generative AI to provide near real-time search and analysis with relevance to reduce your time to value.
 		>
 		>Elastic offers the following solutions or types of projects:
 		>
-		>* [Elasticsearch](https://www.elastic.co/docs/solutions/search): Build powerful search and RAG applications using Elasticsearch's vector database, AI toolkit, and advanced retrieval capabilities.  
+		>* [Elasticsearch](https://www.elastic.co/docs/solutions/search): Build powerful search and RAG applications using Elasticsearch's vector database, AI toolkit, and advanced retrieval capabilities.
 		>* [Elastic Observability](https://www.elastic.co/docs/solutions/observability): Gain comprehensive visibility into applications, infrastructure, and user experience through logs, metrics, traces, and other telemetry data, all in a single interface.
 		>* [Elastic Security](https://www.elastic.co/docs/solutions/security): Combine SIEM, endpoint security, and cloud security to provide comprehensive tools for threat detection and prevention, investigation, and response.
-		
+
 		The documentation is organized to guide you through your journey with Elastic, from learning the basics to deploying and managing complex solutions. Here is a detailed breakdown of the documentation structure:
-		
-		* [**Elastic fundamentals**](https://www.elastic.co/docs/get-started): Understand the basics about the deployment options, platform, and solutions, and features of the documentation.  
-		* [**Solutions and use cases**](https://www.elastic.co/docs/solutions): Learn use cases, evaluate, and implement Elastic's solutions: Observability, Search, and Security.  
-		* [**Manage data**](https://www.elastic.co/docs/manage-data): Learn about data store primitives, ingestion and enrichment, managing the data lifecycle, and migrating data.  
-		* [**Explore and analyze**](https://www.elastic.co/docs/explore-analyze): Get value from data through querying, visualization, machine learning, and alerting.  
-		* [**Deploy and manage**](https://www.elastic.co/docs/deploy-manage): Deploy and manage production-ready clusters. Covers deployment options and maintenance tasks.  
-		* [**Manage your Cloud account**](https://www.elastic.co/docs/cloud-account): A dedicated section for user-facing cloud account tasks like resetting passwords.  
-		* [**Troubleshoot**](https://www.elastic.co/docs/troubleshoot): Identify and resolve problems.  
-		* [**Extend and contribute**](https://www.elastic.co/docs/extend): How to contribute to or integrate with Elastic, from open source to plugins to integrations.  
-		* [**Release notes**](https://www.elastic.co/docs/release-notes): Contains release notes and changelogs for each new release.  
+
+		* [**Elastic fundamentals**](https://www.elastic.co/docs/get-started): Understand the basics about the deployment options, platform, and solutions, and features of the documentation.
+		* [**Solutions and use cases**](https://www.elastic.co/docs/solutions): Learn use cases, evaluate, and implement Elastic's solutions: Observability, Search, and Security.
+		* [**Manage data**](https://www.elastic.co/docs/manage-data): Learn about data store primitives, ingestion and enrichment, managing the data lifecycle, and migrating data.
+		* [**Explore and analyze**](https://www.elastic.co/docs/explore-analyze): Get value from data through querying, visualization, machine learning, and alerting.
+		* [**Deploy and manage**](https://www.elastic.co/docs/deploy-manage): Deploy and manage production-ready clusters. Covers deployment options and maintenance tasks.
+		* [**Manage your Cloud account**](https://www.elastic.co/docs/cloud-account): A dedicated section for user-facing cloud account tasks like resetting passwords.
+		* [**Troubleshoot**](https://www.elastic.co/docs/troubleshoot): Identify and resolve problems.
+		* [**Extend and contribute**](https://www.elastic.co/docs/extend): How to contribute to or integrate with Elastic, from open source to plugins to integrations.
+		* [**Release notes**](https://www.elastic.co/docs/release-notes): Contains release notes and changelogs for each new release.
 		* [**Reference**](https://www.elastic.co/docs/reference): Reference material for core tasks and manuals for optional products.
 		""";
 

src/Elastic.Markdown/Exporters/LlmMarkdownExporter.cs

@@ -62,4 +62,5 @@ public record SearchResultItem
 	public required string Description { get; init; }
 	public required SearchResultItemParent[] Parents { get; init; }
 	public float Score { get; init; }
+	public string? HighlightedBody { get; init; }
 }