diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e622aa21d..69ee15caf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,17 +94,7 @@ jobs: run: dotnet run --project build -c release -- unit-test - name: Publish AOT - if: ${{ matrix.os != 'ubuntu-latest' }} # publish containers already validates AOT build run: dotnet run --project build -c release -- publishbinaries - - - name: Publish Containers - if: ${{ matrix.os == 'ubuntu-latest' }} - run: dotnet run --project build -c release -- publishcontainers - - - name: Run Container - if: ${{ matrix.os == 'ubuntu-latest' }} - run: docker run elastic/docs-builder:ci-${{ github.event.pull_request.number }} --help - integration: runs-on: docs-builder-latest-16 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 596ef89bf..0cb2220b4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -67,7 +67,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Publish Containers - run: ./build.sh publishcontainers + run: ./build.sh publishcontainers\ build-lambda: needs: diff --git a/Directory.Packages.props b/Directory.Packages.props index 969de380c..dcb97bed9 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -23,9 +23,8 @@ - - + diff --git a/build/Targets.fs b/build/Targets.fs index b82b6c7fe..47baaf75a 100644 --- a/build/Targets.fs +++ b/build/Targets.fs @@ -77,40 +77,32 @@ let private publishZip _ = let private publishContainers _ = let createImage project = - let ci = Environment.environVarOrNone "GITHUB_ACTIONS" - let pr = - match Environment.environVarOrNone "GITHUB_REF_NAME" with - | None -> None - | Some s when s.EndsWith "/merge" -> Some (s.Split('/') |> Seq.head) - | _ -> None let imageTag = match project with - | _ -> "9.0-noble-chiseled-aot" + | "docs-builder" -> "jammy-chiseled-aot" + | _ -> "jammy-chiseled-aot" let labels = let exitCode = exec { validExitCode (fun _ -> true) exit_code_of "git" "describe" "--tags" "--exact-match" "HEAD" } - match (exitCode, pr) with - | 0, _ -> "edge;latest" - | _, None -> "edge" - | _, Some pr -> $"ci-%s{pr}" + match exitCode with | 0 -> "edge;latest" | _ -> "edge" let args = ["publish"; $"src/tooling/%s{project}/%s{project}.csproj"] @ [ "/t:PublishContainer"; "-p"; "DebugType=none"; - "-p"; $"ContainerBaseImage=mcr.microsoft.com/dotnet/nightly/runtime-deps:%s{imageTag}"; + "-p"; $"ContainerBaseImage=mcr.microsoft.com/dotnet/nightly/runtime-deps:8.0-%s{imageTag}"; "-p"; $"ContainerImageTags=\"%s{labels};%s{Software.Version.Normalize()}\"" "-p"; $"ContainerRepository=elastic/%s{project}" ] let registry = - match (ci, pr) with - | Some _, None -> [ - "-p"; "ContainerRegistry=ghcr.io" - "-p"; "ContainerUser=1001:1001"; - ] - | _, _ -> [] + match Environment.environVarOrNone "GITHUB_ACTIONS" with + | None -> [] + | Some _ -> [ + "-p"; "ContainerRegistry=ghcr.io" + "-p"; "ContainerUser=1001:1001"; + ] exec { run "dotnet" (args @ registry) } createImage "docs-builder" createImage "docs-assembler" diff --git a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/AskAiSuggestions.tsx b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/AskAiSuggestions.tsx index 7399095dd..0af28a82c 100644 --- a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/AskAiSuggestions.tsx +++ b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/AskAi/AskAiSuggestions.tsx @@ -1,11 +1,5 @@ import { useSearchActions, useSearchTerm } from '../search.store' -import { - EuiButton, - EuiIcon, - EuiSpacer, - EuiText, - useEuiTheme, -} from '@elastic/eui' +import { EuiButton, EuiSpacer, EuiText, useEuiTheme } from '@elastic/eui' import { css } from '@emotion/react' import * as React from 'react' @@ -32,16 +26,7 @@ export const AskAiSuggestions = (props: Props) => { ` return ( <> -
- - Ask Elastic Docs AI Assistant -
+ Ask Elastic Docs AI Assistant {searchTerm && ( { const searchTerm = useSearchTerm() - const [activePage, setActivePage] = useState(0) - const debouncedSearchTerm = useDebounce(searchTerm, 300) - useEffect(() => { - setActivePage(0) - }, [debouncedSearchTerm]) - const { data, error, isLoading, isFetching } = useSearchQuery({ - searchTerm, - pageNumber: activePage + 1, - }) + const { data, error, isLoading } = useSearchQuery() const { euiTheme } = useEuiTheme() if (!searchTerm) { @@ -37,193 +23,88 @@ export const SearchResults = () => { return
Error loading search results: {error.message}
} - return ( -
-
- {isLoading || isFetching ? ( - - ) : ( - - )} - - Search results for{' '} - - {searchTerm} - - + if (isLoading) { + return ( +
+ Loading search results...
- - {data && ( - <> -
    - {data.results.map((result) => ( - - ))} -
-
- - setActivePage(activePage) - } - /> -
- - )} -
- ) -} - -interface SearchResultListItemProps { - item: SearchResultItem -} - -function SearchResultListItem({ item: result }: SearchResultListItemProps) { - const { euiTheme } = useEuiTheme() - const searchTerm = useSearchTerm() - const highlightSearchTerms = useMemo( - () => searchTerm.toLowerCase().split(' '), - [searchTerm] - ) + ) + } - if (highlightSearchTerms.includes('esql')) { - highlightSearchTerms.push('es|ql') + if (!data || data.results.length === 0) { + return No results found for "{searchTerm}" } - if (highlightSearchTerms.includes('dotnet')) { - highlightSearchTerms.push('.net') + const buttonCss = css` + border: none; + vertical-align: top; + justify-content: flex-start; + block-size: 100%; + padding-block: 4px; + & > span { + justify-content: flex-start; + align-items: flex-start; + } + svg { + color: ${euiTheme.colors.textSubdued}; + } + .euiIcon { + margin-top: 4px; + } + ` + + const trimDescription = (description: string) => { + const limit = 200 + return description.length > limit + ? description.slice(0, limit) + '...' + : description } - return ( -
  • -
    - -
    - - - {result.title} - - - -
    -
    -
    • - ) -} -function Breadcrumbs({ - parents, - highlightSearchTerms, -}: { - parents: SearchResultItem['parents'] - highlightSearchTerms: string[] -}) { - const { euiTheme } = useEuiTheme() - const { fontSize: smallFontsize } = useEuiFontSize('xs') return ( -
      - {parents - .slice(1) // skip /docs - .map((parent) => ( -
    • - - Search Results for "{searchTerm}" + +
        + {data.results.map((result) => ( +
      • + +
        - + - {parent.title} - - - + {trimDescription(result.description)} + +
        +         + {/**/} + {/*{result.title}*/}
        • ))} -
      +
    +
    ) } diff --git a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/Search/useSearchQuery.ts b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/Search/useSearchQuery.ts index f3ab3ad5f..d9cd3455f 100644 --- a/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/Search/useSearchQuery.ts +++ b/src/Elastic.Documentation.Site/Assets/web-components/SearchOrAskAi/Search/useSearchQuery.ts @@ -1,56 +1,36 @@ -import { keepPreviousData, useQuery } from '@tanstack/react-query' +import { useSearchTerm } from '../search.store' +import { useQuery } from '@tanstack/react-query' import { useDebounce } from '@uidotdev/usehooks' import * as z from 'zod' -const SearchResultItemParent = z.object({ - url: z.string(), - title: z.string(), -}) - const SearchResultItem = z.object({ url: z.string(), title: z.string(), description: z.string(), score: z.number(), - parents: z.array(SearchResultItemParent), }) -export type SearchResultItem = z.infer - const SearchResponse = z.object({ results: z.array(SearchResultItem), totalResults: z.number(), - pageCount: z.number(), - pageNumber: z.number(), - pageSize: z.number(), }) -export type SearchResponse = z.infer - -type Props = { - searchTerm: string - pageNumber?: number -} +type SearchResponse = z.infer -export const useSearchQuery = ({ searchTerm, pageNumber = 1 }: Props) => { +export const useSearchQuery = () => { + const searchTerm = useSearchTerm() const trimmedSearchTerm = searchTerm.trim() const debouncedSearchTerm = useDebounce(trimmedSearchTerm, 300) return useQuery({ - queryKey: [ - 'search', - { searchTerm: debouncedSearchTerm.toLowerCase(), pageNumber }, - ], + queryKey: ['search', { searchTerm: debouncedSearchTerm }], queryFn: async () => { if (!debouncedSearchTerm || debouncedSearchTerm.length < 1) { return SearchResponse.parse({ results: [], totalResults: 0 }) } - const params = new URLSearchParams({ - q: debouncedSearchTerm, - page: pageNumber.toString(), - }) const response = await fetch( - '/docs/_api/v1/search?' + params.toString() + '/docs/_api/v1/search?q=' + + encodeURIComponent(debouncedSearchTerm) ) if (!response.ok) { throw new Error( @@ -62,7 +42,6 @@ export const useSearchQuery = ({ searchTerm, pageNumber = 1 }: Props) => { }, enabled: !!trimmedSearchTerm && trimmedSearchTerm.length >= 1, refetchOnWindowFocus: false, - placeholderData: keepPreviousData, - staleTime: 1000 * 60 * 5, // 5 minutes + staleTime: 1000 * 60 * 10, // 10 minutes }) } diff --git a/src/Elastic.Documentation/Search/DocumentationDocument.cs b/src/Elastic.Documentation/Search/DocumentationDocument.cs index 28c23ee3f..df036320d 100644 --- a/src/Elastic.Documentation/Search/DocumentationDocument.cs +++ b/src/Elastic.Documentation/Search/DocumentationDocument.cs @@ -7,15 +7,6 @@ namespace Elastic.Documentation.Search; -public record ParentDocument -{ - [JsonPropertyName("title")] - public string? Title { get; set; } - - [JsonPropertyName("url")] - public string? Url { get; set; } -} - public record DocumentationDocument { [JsonPropertyName("title")] @@ -39,12 +30,6 @@ public record DocumentationDocument [JsonPropertyName("body")] public string? Body { get; set; } - [JsonPropertyName("url_segment_count")] - public int? UrlSegmentCount { get; set; } - [JsonPropertyName("abstract")] public string? Abstract { get; set; } - - [JsonPropertyName("parents")] - public ParentDocument[] Parents { get; set; } = []; } diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs index 49168cca1..723e8f32b 100644 --- a/src/Elastic.Markdown/DocumentationGenerator.cs +++ b/src/Elastic.Markdown/DocumentationGenerator.cs @@ -262,8 +262,7 @@ private async Task ProcessFile(HashSet offendingFiles, DocumentationFile Resolvers = DocumentationSet.MarkdownParser.Resolvers, Document = document, SourceFile = markdown, - DefaultOutputFile = outputFile, - DocumentationSet = DocumentationSet + DefaultOutputFile = outputFile }, ctx); } } diff --git a/src/Elastic.Markdown/Exporters/IMarkdownExporter.cs b/src/Elastic.Markdown/Exporters/IMarkdownExporter.cs index 492df8ced..570544141 100644 --- a/src/Elastic.Markdown/Exporters/IMarkdownExporter.cs +++ b/src/Elastic.Markdown/Exporters/IMarkdownExporter.cs @@ -18,7 +18,6 @@ public record MarkdownExportFileContext public required MarkdownFile SourceFile { get; init; } public required IFileInfo DefaultOutputFile { get; init; } public string? LLMText { get; set; } - public required DocumentationSet DocumentationSet { get; init; } } public interface IMarkdownExporter diff --git a/src/Elastic.Markdown/IO/DocumentationSet.cs b/src/Elastic.Markdown/IO/DocumentationSet.cs index e848ae8a6..7c8c5784d 100644 --- a/src/Elastic.Markdown/IO/DocumentationSet.cs +++ b/src/Elastic.Markdown/IO/DocumentationSet.cs @@ -78,9 +78,7 @@ INavigationItem[] GetParents(INavigationItem current) { if (parent is null) continue; - if (parents.All(i => i.Url != parent.Url)) - parents.Add(parent); - + parents.Add(parent); parent = parent.Parent; } while (parent != null); diff --git a/src/api/Elastic.Documentation.Api.Core/Search/SearchUsecase.cs b/src/api/Elastic.Documentation.Api.Core/Search/SearchUsecase.cs index 8114dd475..256ea1049 100644 --- a/src/api/Elastic.Documentation.Api.Core/Search/SearchUsecase.cs +++ b/src/api/Elastic.Documentation.Api.Core/Search/SearchUsecase.cs @@ -8,7 +8,6 @@ public class SearchUsecase(ISearchGateway searchGateway) { public async Task Search(SearchRequest request, Cancel ctx = default) { - // var validationResult = validator.Validate(request); // if (!validationResult.IsValid) // throw new ArgumentException(validationResult.Message); @@ -16,17 +15,13 @@ public async Task Search(SearchRequest request, Cancel ctx = def var (totalHits, results) = await searchGateway.SearchAsync( request.Query, request.PageNumber, - request.PageSize, - ctx + request.PageSize, ctx ); - return new SearchResponse { Results = results, - TotalResults = totalHits, - PageNumber = request.PageNumber, - PageSize = request.PageSize, + TotalResults = totalHits }; } } @@ -35,24 +30,13 @@ public record SearchRequest { public required string Query { get; init; } public int PageNumber { get; init; } = 1; - public int PageSize { get; init; } = 5; + public int PageSize { get; init; } = 10; } public record SearchResponse { public required IEnumerable Results { get; init; } public required int TotalResults { get; init; } - public required int PageNumber { get; init; } - public required int PageSize { get; init; } - public int PageCount => TotalResults > 0 - ? (int)Math.Ceiling((double)TotalResults / PageSize) - : 0; -} - -public record SearchResultItemParent -{ - public required string Title { get; init; } - public required string Url { get; init; } } public record SearchResultItem @@ -60,6 +44,5 @@ public record SearchResultItem public required string Url { get; init; } public required string Title { get; init; } public required string Description { get; init; } - public required SearchResultItemParent[] Parents { get; init; } - public float Score { get; init; } + public required double Score { get; init; } } diff --git a/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/AskAi/LlmGatewayAskAiGateway.cs b/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/AskAi/LlmGatewayAskAiGateway.cs index 3430ecec5..fb7236c72 100644 --- a/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/AskAi/LlmGatewayAskAiGateway.cs +++ b/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/AskAi/LlmGatewayAskAiGateway.cs @@ -40,7 +40,7 @@ string ThreadId public static LlmGatewayRequest CreateFromRequest(AskAiRequest request) => new( UserContext: new UserContext("elastic-docs-v3@invalid"), - PlatformContext: new PlatformContext("docs_site", "docs_assistant", []), + PlatformContext: new PlatformContext("support_portal", "support_assistant", []), Input: [ new ChatInput("user", AskAiRequest.SystemPrompt), diff --git a/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/Search/ElasticsearchGateway.cs b/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/Search/ElasticsearchGateway.cs deleted file mode 100644 index 0f14f4f2f..000000000 --- a/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/Search/ElasticsearchGateway.cs +++ /dev/null @@ -1,196 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using System.Text.Json.Serialization; -using Elastic.Clients.Elasticsearch; -using Elastic.Clients.Elasticsearch.QueryDsl; -using Elastic.Clients.Elasticsearch.Serialization; -using Elastic.Documentation.Api.Core.Search; -using Elastic.Transport; -using Microsoft.Extensions.Logging; - -namespace Elastic.Documentation.Api.Infrastructure.Adapters.Search; - -internal sealed record DocumentDto -{ - [JsonPropertyName("title")] - public required string Title { get; init; } - - [JsonPropertyName("url")] - public required string Url { get; init; } - - [JsonPropertyName("description")] - public string? Description { get; init; } - - [JsonPropertyName("body")] - public string? Body { get; init; } - - [JsonPropertyName("abstract")] - public required string Abstract { get; init; } - - [JsonPropertyName("url_segment_count")] - public int UrlSegmentCount { get; init; } - - [JsonPropertyName("parents")] - public ParentDocumentDto[] Parents { get; init; } = []; -} - -internal sealed record ParentDocumentDto -{ - [JsonPropertyName("title")] - public required string Title { get; init; } - - [JsonPropertyName("url")] - public required string Url { get; init; } -} - -public class ElasticsearchGateway : ISearchGateway -{ - private readonly ElasticsearchClient _client; - private readonly ElasticsearchOptions _elasticsearchOptions; - private readonly ILogger _logger; - - public ElasticsearchGateway(ElasticsearchOptions elasticsearchOptions, ILogger logger) - { - _logger = logger; - _elasticsearchOptions = elasticsearchOptions; - var nodePool = new SingleNodePool(new Uri(elasticsearchOptions.Url.Trim())); - var clientSettings = new ElasticsearchClientSettings( - nodePool, - sourceSerializer: (_, settings) => new DefaultSourceSerializer(settings, EsJsonContext.Default) - ) - .DefaultIndex(elasticsearchOptions.IndexName) - .Authentication(new ApiKey(elasticsearchOptions.ApiKey)); - - _client = new ElasticsearchClient(clientSettings); - } - - public async Task<(int TotalHits, List Results)> SearchAsync(string query, int pageNumber, int pageSize, Cancel ctx = default) => - await ExactSearchAsync(query, pageNumber, pageSize, ctx); - - public async Task<(int TotalHits, List Results)> ExactSearchAsync(string query, int pageNumber, int pageSize, Cancel ctx = default) - { - _logger.LogInformation("Starting search for '{Query}' with pageNumber={PageNumber}, pageSize={PageSize}", query, pageNumber, pageSize); - - var searchQuery = query.Replace("dotnet", "net", StringComparison.InvariantCultureIgnoreCase); - - try - { - var response = await _client.SearchAsync(s => s - .Indices(_elasticsearchOptions.IndexName) - .Query(q => q - .Bool(b => b - .Should( - // Tier 1: Exact/Prefix matches (highest boost) - sh => sh.Prefix(p => p - .Field("title.keyword") - .Value(searchQuery) - .CaseInsensitive(true) - .Boost(300.0f) - ), - - // Tier 2: Semantic search (combined into one clause) - sh => sh.DisMax(dm => dm - .Queries( - dq => dq.Semantic(sem => sem - .Field("title.semantic_text") - .Query(searchQuery) - ), - dq => dq.Semantic(sem => sem - .Field("abstract") - .Query(searchQuery) - ) - ) - .Boost(200.0f) - ), - - // Tier 3: Standard text matching - sh => sh.DisMax(dm => dm - .Queries( - dq => dq.MatchBoolPrefix(m => m - .Field(f => f.Title) - .Query(searchQuery) - ), - dq => dq.Match(m => m - .Field(f => f.Title) - .Query(searchQuery) - .Operator(Operator.And) - ), - dq => dq.Match(m => m - .Field(f => f.Abstract) - .Query(searchQuery) - ) - ) - .Boost(100.0f) - ), - - // Tier 4: Parent matching - sh => sh.Match(m => m - .Field("parents.title") - .Query(searchQuery) - .Boost(75.0f) - ), - - // Tier 5: Fuzzy fallback - sh => sh.Match(m => m - .Field(f => f.Title) - .Query(searchQuery) - .Fuzziness(1) // Reduced from 2 - .Boost(25.0f) - ) - ) - .MustNot(mn => mn.Terms(t => t - .Field("url.keyword") - .Terms(factory => factory.Value("/docs", "/docs/", "/docs/404", "/docs/404/")) - )) - .MinimumShouldMatch(1) - ) - ) - .From((pageNumber - 1) * pageSize) - .Size(pageSize), ctx); - - if (!response.IsValidResponse) - { - _logger.LogWarning("Elasticsearch search response was not valid. Reason: {Reason}", - response.ElasticsearchServerError?.Error?.Reason ?? "Unknown"); - } - else - { - _logger.LogInformation("Search completed for '{Query}'. Total hits: {TotalHits}", query, response.Total); - } - - return ProcessSearchResponse(response); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error occurred during Elasticsearch search for '{Query}'", query); - throw; - } - } - - - private static (int TotalHits, List Results) ProcessSearchResponse(SearchResponse response) - { - var totalHits = (int)response.Total; - - var results = response.Documents.Select((doc, index) => new SearchResultItem - { - Url = doc.Url, - Title = doc.Title, - Description = doc.Description ?? string.Empty, - Parents = doc.Parents.Select(parent => new SearchResultItemParent - { - Title = parent.Title, - Url = parent.Url - }).ToArray(), - Score = (float)(response.Hits.ElementAtOrDefault(index)?.Score ?? 0.0) - }).ToList(); - - return (totalHits, results); - } -} - -[JsonSerializable(typeof(DocumentDto))] -[JsonSerializable(typeof(ParentDocumentDto))] -internal sealed partial class EsJsonContext : JsonSerializerContext; diff --git a/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/Search/ElasticsearchOptions.cs b/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/Search/ElasticsearchOptions.cs deleted file mode 100644 index d30a3083a..000000000 --- a/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/Search/ElasticsearchOptions.cs +++ /dev/null @@ -1,14 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using Elastic.Documentation.Api.Infrastructure.Aws; - -namespace Elastic.Documentation.Api.Infrastructure.Adapters.Search; - -public class ElasticsearchOptions(IParameterProvider parameterProvider) -{ - public string Url { get; } = parameterProvider.GetParam("docs-elasticsearch-url").GetAwaiter().GetResult(); - public string ApiKey { get; } = parameterProvider.GetParam("docs-elasticsearch-apikey").GetAwaiter().GetResult(); - public string IndexName { get; } = parameterProvider.GetParam("docs-elasticsearch-index").GetAwaiter().GetResult() ?? "documentation-latest"; -} diff --git a/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/Search/MockSearchGateway.cs b/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/Search/MockSearchGateway.cs index 2e5705986..608b83201 100644 --- a/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/Search/MockSearchGateway.cs +++ b/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/Search/MockSearchGateway.cs @@ -16,14 +16,14 @@ public class MockSearchGateway : ISearchGateway Title = "Kibana: Explore, Visualize, Discover Data", Description = "Run data analytics at speed and scale for observability, security, and search with Kibana. Powerful analysis on any data from any source.", - Parents = [] + Score = 0.92 }, new SearchResultItem { Url = "https://www.elastic.co/docs/explore-analyze", Title = "Explore and analyze | Elastic Docs", Description = "Kibana provides a comprehensive suite of tools to help you search, interact with, explore, and analyze your data effectively.", - Parents = [] + Score = 0.86 }, new SearchResultItem { @@ -31,7 +31,7 @@ public class MockSearchGateway : ISearchGateway Title = "Install Kibana | Elastic Docs", Description = "Information on how to set up Kibana and get it running, including downloading, enrollment with Elasticsearch cluster, and configuration.", - Parents = [] + Score = 0.75 }, new SearchResultItem { @@ -39,7 +39,7 @@ public class MockSearchGateway : ISearchGateway Title = "Kibana Lens – Data visualization. Simply.", Description = "Kibana Lens simplifies the process of data visualization through a drag‑and‑drop experience, ideal for exploring logs, trends, and metrics.", - Parents = [] + Score = 0.70 }, new SearchResultItem { @@ -47,7 +47,7 @@ public class MockSearchGateway : ISearchGateway Title = "Elastic Docs – Elastic products, guides & reference", Description = "Official Elastic documentation. Explore guides for Elastic Cloud (hosted & on‑prem), product documentation, how‑to guides and API reference.", - Parents = [] + Score = 0.88 }, new SearchResultItem { @@ -55,14 +55,14 @@ public class MockSearchGateway : ISearchGateway Title = "Get started | Elastic Docs", Description = "Use Elasticsearch to search, index, store, and analyze data of all shapes and sizes in near real time. Kibana is the graphical user interface for Elasticsearch.", - Parents = [] + Score = 0.85 }, new SearchResultItem { Url = "https://www.elastic.co/docs/solutions/search/elasticsearch-basics-quickstart", Title = "Elasticsearch basics quickstart", Description = "Hands‑on introduction to fundamental Elasticsearch concepts: indices, documents, mappings, and search via Console syntax.", - Parents = [] + Score = 0.80 }, new SearchResultItem { @@ -70,26 +70,21 @@ public class MockSearchGateway : ISearchGateway Title = "Elasticsearch API documentation", Description = "Elastic provides REST APIs that are used by the UI components and can be called directly to configure and access Elasticsearch features.", - Parents = [] + Score = 0.78 } ]; public async Task<(int TotalHits, List Results)> SearchAsync(string query, int pageNumber, int pageSize, CancellationToken ctx = default) { var filteredResults = Results - .Where(item => - item.Title.Equals(query, StringComparison.OrdinalIgnoreCase) || - item.Description?.Equals(query, StringComparison.OrdinalIgnoreCase) == true) - .ToList(); - - var pagedResults = filteredResults + .Where(item => query.Split(' ') + .All(token => item.Title.Contains(token, StringComparison.OrdinalIgnoreCase) || + item.Description.Contains(token, StringComparison.OrdinalIgnoreCase))) .Skip((pageNumber - 1) * pageSize) .Take(pageSize) .ToList(); - Console.WriteLine($"MockSearchGateway: Paged results count: {pagedResults.Count}"); - return await Task.Delay(1000, ctx) - .ContinueWith(_ => (TotalHits: filteredResults.Count, Results: pagedResults), ctx); + .ContinueWith(_ => (TotalHits: filteredResults.Count, Results: filteredResults), ctx); } } diff --git a/src/api/Elastic.Documentation.Api.Infrastructure/Aws/LocalParameterProvider.cs b/src/api/Elastic.Documentation.Api.Infrastructure/Aws/LocalParameterProvider.cs index f4fea9f47..66d817d61 100644 --- a/src/api/Elastic.Documentation.Api.Infrastructure/Aws/LocalParameterProvider.cs +++ b/src/api/Elastic.Documentation.Api.Infrastructure/Aws/LocalParameterProvider.cs @@ -12,7 +12,10 @@ public async Task GetParam(string name, bool withDecryption = true, Canc { case "llm-gateway-service-account": { - var serviceAccountKeyPath = GetEnv("LLM_GATEWAY_SERVICE_ACCOUNT_KEY_PATH"); + const string envName = "LLM_GATEWAY_SERVICE_ACCOUNT_KEY_PATH"; + var serviceAccountKeyPath = Environment.GetEnvironmentVariable(envName); + if (string.IsNullOrEmpty(serviceAccountKeyPath)) + throw new ArgumentException($"Environment variable '{envName}' not found."); if (!File.Exists(serviceAccountKeyPath)) throw new ArgumentException($"Service account key file not found at '{serviceAccountKeyPath}'."); var serviceAccountKey = await File.ReadAllTextAsync(serviceAccountKeyPath, ctx); @@ -20,19 +23,11 @@ public async Task GetParam(string name, bool withDecryption = true, Canc } case "llm-gateway-function-url": { - return GetEnv("LLM_GATEWAY_FUNCTION_URL"); - } - case "docs-elasticsearch-url": - { - return GetEnv("DOCUMENTATION_ELASTIC_URL"); - } - case "docs-elasticsearch-apikey": - { - return GetEnv("DOCUMENTATION_ELASTIC_APIKEY"); - } - case "docs-elasticsearch-index": - { - return "semantic-documentation-latest"; + const string envName = "LLM_GATEWAY_FUNCTION_URL"; + var value = Environment.GetEnvironmentVariable(envName); + if (string.IsNullOrEmpty(value)) + throw new ArgumentException($"Environment variable '{envName}' not found."); + return value; } default: { @@ -40,12 +35,4 @@ public async Task GetParam(string name, bool withDecryption = true, Canc } } } - - private static string GetEnv(string name) - { - var value = Environment.GetEnvironmentVariable(name); - if (string.IsNullOrEmpty(value)) - throw new ArgumentException($"Environment variable '{name}' not found."); - return value; - } } diff --git a/src/api/Elastic.Documentation.Api.Infrastructure/Elastic.Documentation.Api.Infrastructure.csproj b/src/api/Elastic.Documentation.Api.Infrastructure/Elastic.Documentation.Api.Infrastructure.csproj index c7fc40d3a..effd23891 100644 --- a/src/api/Elastic.Documentation.Api.Infrastructure/Elastic.Documentation.Api.Infrastructure.csproj +++ b/src/api/Elastic.Documentation.Api.Infrastructure/Elastic.Documentation.Api.Infrastructure.csproj @@ -16,8 +16,7 @@ - - + diff --git a/src/api/Elastic.Documentation.Api.Infrastructure/MappingsExstension.cs b/src/api/Elastic.Documentation.Api.Infrastructure/MappingsExstension.cs index 1dcede6d9..fadc1afd5 100644 --- a/src/api/Elastic.Documentation.Api.Infrastructure/MappingsExstension.cs +++ b/src/api/Elastic.Documentation.Api.Infrastructure/MappingsExstension.cs @@ -32,21 +32,14 @@ private static void MapAskAiEndpoint(IEndpointRouteBuilder group) private static void MapSearchEndpoint(IEndpointRouteBuilder group) { var searchGroup = group.MapGroup("/search"); - _ = searchGroup.MapGet("/", - async ( - [FromQuery(Name = "q")] string query, - [FromQuery(Name = "page")] int? pageNumber, - SearchUsecase searchUsecase, - Cancel ctx - ) => + _ = searchGroup.MapGet("/", async ([FromQuery(Name = "q")] string query, SearchUsecase searchUsecase, Cancel ctx) => + { + var searchRequest = new SearchRequest { - var searchRequest = new SearchRequest - { - Query = query, - PageNumber = pageNumber ?? 1 - }; - var searchResponse = await searchUsecase.Search(searchRequest, ctx); - return Results.Ok(searchResponse); - }); + Query = query + }; + var searchResponse = await searchUsecase.Search(searchRequest, ctx); + return Results.Ok(searchResponse); + }); } } diff --git a/src/api/Elastic.Documentation.Api.Infrastructure/ServicesExtension.cs b/src/api/Elastic.Documentation.Api.Infrastructure/ServicesExtension.cs index 293547b92..43862f029 100644 --- a/src/api/Elastic.Documentation.Api.Infrastructure/ServicesExtension.cs +++ b/src/api/Elastic.Documentation.Api.Infrastructure/ServicesExtension.cs @@ -113,8 +113,7 @@ private static void AddSearchUsecase(IServiceCollection services, AppEnv appEnv) { var logger = GetLogger(services); logger?.LogInformation("Configuring Search use case for environment {AppEnvironment}", appEnv); - _ = services.AddScoped(); - _ = services.AddScoped(); + _ = services.AddScoped(); _ = services.AddScoped(); } } diff --git a/src/tooling/Elastic.Documentation.Tooling/Exporters/ElasticsearchMarkdownExporter.cs b/src/tooling/Elastic.Documentation.Tooling/Exporters/ElasticsearchMarkdownExporter.cs index 20499b6f9..e5bff6804 100644 --- a/src/tooling/Elastic.Documentation.Tooling/Exporters/ElasticsearchMarkdownExporter.cs +++ b/src/tooling/Elastic.Documentation.Tooling/Exporters/ElasticsearchMarkdownExporter.cs @@ -12,10 +12,8 @@ using Elastic.Ingest.Elasticsearch.Catalog; using Elastic.Ingest.Elasticsearch.Semantic; using Elastic.Markdown.Exporters; -using Elastic.Markdown.IO; using Elastic.Transport; using Elastic.Transport.Products.Elasticsearch; -using Markdig.Syntax; using Microsoft.Extensions.Logging; namespace Elastic.Documentation.Tooling.Exporters; @@ -35,7 +33,6 @@ public class ElasticsearchMarkdownExporter(ILoggerFactory logFactory, IDiagnosti /// protected override CatalogIndexChannel NewChannel(CatalogIndexChannelOptions options) => new(options); } - public class ElasticsearchMarkdownSemanticExporter(ILoggerFactory logFactory, IDiagnosticsCollector collector, DocumentationEndpoints endpoints) : ElasticsearchMarkdownExporterBase, SemanticIndexChannel> (logFactory, collector, endpoints) @@ -44,11 +41,10 @@ public class ElasticsearchMarkdownSemanticExporter(ILoggerFactory logFactory, ID protected override SemanticIndexChannelOptions NewOptions(DistributedTransport transport) => new(transport) { GetMapping = (inferenceId, _) => CreateMapping(inferenceId), - GetMappingSettings = (_, _) => CreateMappingSetting(), IndexFormat = "semantic-documentation-{0:yyyy.MM.dd.HHmmss}", ActiveSearchAlias = "semantic-documentation", IndexNumThreads = IndexNumThreads, - InferenceCreateTimeout = TimeSpan.FromMinutes(4) + InferenceCreateTimeout = TimeSpan.FromMinutes(4), }; /// @@ -71,87 +67,34 @@ public abstract class ElasticsearchMarkdownExporterBase 8; - protected static string CreateMappingSetting() => - // language=json - """ - { - "analysis": { - "analyzer": { - "synonyms_analyzer": { - "tokenizer": "whitespace", - "filter": [ - "lowercase", - "synonyms_filter" - ] - } - }, - "filter": { - "synonyms_filter": { - "type": "synonym", - "synonyms_set": "docs", - "updateable": true - } - } - } - } - """; - protected static string CreateMapping(string? inferenceId) => // langugage=json $$""" - { - "properties": { - "title": { - "type": "text", - "search_analyzer": "synonyms_analyzer", - "fields": { - "keyword": { - "type": "keyword" - } - {{(!string.IsNullOrWhiteSpace(inferenceId) ? $$""", "semantic_text": {{{InferenceMapping(inferenceId)}}}""" : "")}} - } - }, - "url": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - } - }, - "url_segment_count": { - "type": "integer" - }, - "body": { - "type": "text" - } - {{(!string.IsNullOrWhiteSpace(inferenceId) ? AbstractInferenceMapping(inferenceId) : AbstractMapping())}} - } + { + "properties": { + "title": { "type": "text" }, + "body": { "type": "text" } + {{(!string.IsNullOrWhiteSpace(inferenceId) ? AbstractInferenceMapping(inferenceId) : AbstractMapping())}} } - """; + } + """; private static string AbstractMapping() => // langugage=json """ , "abstract": { - "type": "text" + "type": "text", } """; - private static string InferenceMapping(string inferenceId) => - // langugage=json - $""" - "type": "semantic_text", - "inference_id": "{inferenceId}" - """; - private static string AbstractInferenceMapping(string inferenceId) => // langugage=json $$""" - , "abstract": { - {{InferenceMapping(inferenceId)}} - } - """; + , "abstract": { + "type": "semantic_text", + "inference_id": "{{inferenceId}}" + } + """; public async ValueTask StartAsync(Cancel ctx = default) { @@ -169,7 +112,6 @@ public async ValueTask StartAsync(Cancel ctx = default) }; var transport = new DistributedTransport(configuration); - //The max num threads per allocated node, from testing its best to limit our max concurrency //producing to this number as well var options = NewOptions(transport); @@ -238,41 +180,18 @@ public async ValueTask ExportAsync(MarkdownExportFileContext fileContext, var file = fileContext.SourceFile; var url = file.Url; - if (url is "/docs" or "/docs/404") - { - // Skip the root and 404 pages - _logger.LogInformation("Skipping export for {Url}", url); - return true; - } - - IPositionalNavigation navigation = fileContext.DocumentationSet; - //use LLM text if it was already provided (because we run with both llm and elasticsearch output) - var body = fileContext.LLMText ??= LlmMarkdownExporter.ConvertToLlmMarkdown(document, fileContext.BuildContext); - - var headings = fileContext.Document.Descendants() - .Select(h => (h.GetData("header") as string) ?? string.Empty) - .Where(text => !string.IsNullOrEmpty(text)) - .ToArray(); - + var body = fileContext.LLMText ??= LlmMarkdownExporter.ConvertToLlmMarkdown(fileContext.Document, fileContext.BuildContext); var doc = new DocumentationDocument { Title = file.Title, Url = url, Body = body, Description = fileContext.SourceFile.YamlFrontMatter?.Description, - Abstract = !string.IsNullOrEmpty(body) - ? body[..Math.Min(body.Length, 400)] + " " + string.Join(" \n- ", headings) + ? body[..Math.Min(body.Length, 400)] : string.Empty, Applies = fileContext.SourceFile.YamlFrontMatter?.AppliesTo, - UrlSegmentCount = url.Split('/', StringSplitOptions.RemoveEmptyEntries).Length, - Parents = navigation.GetParentsOfMarkdownFile(file).Select(i => new ParentDocument - { - Title = i.NavigationTitle, - Url = i.Url - }).Reverse().ToArray(), - Headings = headings }; return await TryWrite(doc, ctx); }