Merge branch 'main' into feature/redirect_extensions

Author: cotti

Directory.Packages.props

@@ -16,6 +16,7 @@
     <PackageVersion Include="AWSSDK.Core" Version="4.0.0.2" />
     <PackageVersion Include="AWSSDK.SQS" Version="4.0.0.1" />
     <PackageVersion Include="AWSSDK.S3" Version="4.0.0.1" />
+    <PackageVersion Include="Elastic.Ingest.Elasticsearch" Version="0.11.3" />
   </ItemGroup>
   <!-- Build -->
   <ItemGroup>
@@ -63,4 +64,4 @@
     <PackageVersion Include="xunit.runner.visualstudio" Version="3.0.2" />
     <PackageVersion Include="xunit.v3" Version="1.1.0" />
   </ItemGroup>
-</Project>
+</Project>

install.sh

@@ -7,7 +7,7 @@ ARCH=$(uname -m)
 
 # Map architecture naming
 if [ "$ARCH" = "x86_64" ]; then
-  ARCH="amd64"
+  ARCH="x64"
 elif [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then
   ARCH="arm64"
 fi

src/Elastic.Documentation.Search/Elastic.Documentation.Search.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net9.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <PackageReference Include="Elastic.Ingest.Elasticsearch" />
+    </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Elastic.Documentation\Elastic.Documentation.csproj"/>
+  </ItemGroup>
+
+</Project>

src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs

@@ -35,6 +35,18 @@ public static void EmitWarning(this IDiagnosticsCollector collector, IFileInfo f
 
 	public static void EmitHint(this IDiagnosticsCollector collector, IFileInfo file, string message) =>
 		collector.EmitHint(file.FullName, message);
+
+	/// Emit an error not associated with a file
+	public static void EmitGlobalError(this IDiagnosticsCollector collector, string message, Exception? e = null) =>
+		collector.EmitError(string.Empty, message, e);
+
+	/// Emit a warning not associated with a file
+	public static void EmitGlobalWarning(this IDiagnosticsCollector collector, string message) =>
+		collector.EmitWarning(string.Empty, message);
+
+	/// Emit a hint not associated with a file
+	public static void EmitGlobalHint(this IDiagnosticsCollector collector, string message) =>
+		collector.EmitHint(string.Empty, message);
 }
 
 

src/Elastic.Documentation/Links/LinkRegistry.cs

@@ -2,6 +2,7 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System.Diagnostics.CodeAnalysis;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 using Elastic.Documentation.Serialization;

src/Elastic.Documentation/Search/DocumentationDocument.cs

@@ -0,0 +1,29 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Text.Json.Serialization;
+
+namespace Elastic.Documentation.Search;
+
+public record DocumentationDocument
+{
+	[JsonPropertyName("title")]
+	public string? Title { get; set; }
+
+	[JsonPropertyName("body")]
+	public string? Body { get; set; }
+
+	[JsonPropertyName("abstract")]
+	public string? Abstract { get; set; }
+
+	[JsonPropertyName("headings")]
+	public string[] Headings { get; set; } = [];
+
+	[JsonPropertyName("links")]
+	public string[] Links { get; set; } = [];
+
+	[JsonPropertyName("url")]
+	public string? Url { get; set; }
+}
+

src/Elastic.Documentation/Serialization/SourceGenerationContext.cs

@@ -4,6 +4,7 @@
 
 using System.Text.Json.Serialization;
 using Elastic.Documentation.Links;
+using Elastic.Documentation.Search;
 using Elastic.Documentation.State;
 
 namespace Elastic.Documentation.Serialization;
@@ -16,4 +17,5 @@ namespace Elastic.Documentation.Serialization;
 [JsonSerializable(typeof(GitCheckoutInformation))]
 [JsonSerializable(typeof(LinkRegistry))]
 [JsonSerializable(typeof(LinkRegistryEntry))]
+[JsonSerializable(typeof(DocumentationDocument))]
 public sealed partial class SourceGenerationContext : JsonSerializerContext;

src/Elastic.Markdown/Assets/hljs.ts

@@ -168,7 +168,7 @@ hljs.registerLanguage('esql', function () {
         case_insensitive: true, // language is case-insensitive
         keywords: {
             keyword:
-                'FROM ROW SHOW DISSECT DROP ENRICH EVAL GROK KEEP LIMIT RENAME SORT STATS WHERE METADATA',
+                'FROM ROW SHOW DISSECT DROP ENRICH EVAL GROK KEEP LIMIT LOOKUP JOIN MV_EXPAND RENAME SORT STATS WHERE METADATA CHANGE_POINT',
             literal: ['false', 'true', 'null'],
             function: [
                 // aggregate
@@ -180,30 +180,31 @@ hljs.registerLanguage('esql', function () {
                 'MEDIAN_ABSOLUTE_DEVIATION',
                 'MIN',
                 'PERCENTILE',
+                'ST_CENTROID_AGG',
+                'ST_EXTENT_AGG',
+                'STD_DEV',
                 'SUM',
                 'TOP',
                 'VALUES',
                 'WEIGHTED_AVG',
+                // grouping
                 'BUCKET',
-
+                'CATEGORIZE',
                 // conditional
                 'CASE',
                 'COALESCE',
                 'GREATEST',
                 'LEAST',
-
                 //Date
                 'DATE_DIFF',
                 'DATE_EXTRACT',
                 'DATE_FORMAT',
                 'DATE_PARSE',
                 'DATE_TRUNC',
                 'NOW',
-
                 //ip
                 'CIDR_MATCH',
                 'IP_PREFIX',
-
                 //math
                 'ABS',
                 'ACOS',
@@ -223,18 +224,19 @@ hljs.registerLanguage('esql', function () {
                 'PI',
                 'POW',
                 'ROUND',
+                'ROUND_TO',
+                'SCALB',
                 'SIGNUM',
                 'SIN',
                 'SINH',
                 'SQRT',
                 'TAN',
                 'TANH',
                 'TAU',
-
                 //search
+                'KQL',
                 'MATCH',
                 'QSTR',
-
                 //spatial
                 'ST_DISTANCE',
                 'ST_INTERSECTS',
@@ -243,23 +245,30 @@ hljs.registerLanguage('esql', function () {
                 'ST_WITHIN',
                 'ST_X',
                 'ST_Y',
-
+                'ST_ENVELOPE',
+                'ST_XMAX',
+                'ST_XMIN',
+                'ST_YMAX',
+                'ST_YMIN',
                 //string
-
                 'BIT_LENGTH',
                 'BYTE_LENGTH',
                 'CONCAT',
                 'ENDS_WITH',
                 'FROM_BASE64',
+                'HASH',
                 'LEFT',
                 'LENGTH',
                 'LOCATE',
                 'LTRIM',
+                'MD5',
                 'REPEAT',
                 'REPLACE',
                 'REVERSE',
                 'RIGHT',
                 'RTRIM',
+                'SHA1',
+                'SHA256',
                 'SPACE',
                 'SPLIT',
                 'STARTS_WITH',
@@ -268,12 +277,13 @@ hljs.registerLanguage('esql', function () {
                 'TO_LOWER',
                 'TO_UPPER',
                 'TRIM',
-
                 //type conversion
                 'TO_BOOLEAN',
                 'TO_CARTESIANPOINT',
                 'TO_CARTESIANSHAPE',
+                'TO_DATEPERIOD',
                 'TO_DATETIME',
+                'TO_DATE_NANOS',
                 'TO_DEGREES',
                 'TO_DOUBLE',
                 'TO_GEOPOINT',
@@ -283,8 +293,9 @@ hljs.registerLanguage('esql', function () {
                 'TO_LONG',
                 'TO_RADIANS',
                 'TO_STRING',
+                'TO_TIMEDURATION',
+                'TO_UNSIGNED_LONG',
                 'TO_VERSION',
-
                 //multivalued
                 'MV_APPEND',
                 'MV_AVG',
@@ -303,8 +314,6 @@ hljs.registerLanguage('esql', function () {
                 'MV_SLICE',
                 'MV_SUM',
                 'MV_ZIP',
-
-                'KQL',
             ],
         },
         contains: [

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -18,6 +18,7 @@
 
 namespace Elastic.Markdown;
 
+/// Used primarily for testing, do not use in production paths since it might keep references alive to long
 public interface IConversionCollector
 {
 	void Collect(MarkdownFile file, MarkdownDocument document, string html);
@@ -40,6 +41,7 @@ public class DocumentationGenerator
 	private readonly ILogger _logger;
 	private readonly IFileSystem _writeFileSystem;
 	private readonly IDocumentationFileExporter _documentationFileExporter;
+	private readonly IMarkdownExporter[] _markdownExporters;
 	private HtmlWriter HtmlWriter { get; }
 
 	public DocumentationSet DocumentationSet { get; }
@@ -51,12 +53,14 @@ public DocumentationGenerator(
 		ILoggerFactory logger,
 		INavigationHtmlWriter? navigationHtmlWriter = null,
 		IDocumentationFileOutputProvider? documentationFileOutputProvider = null,
+		IMarkdownExporter[]? markdownExporters = null,
 		IDocumentationFileExporter? documentationExporter = null,
 		IConversionCollector? conversionCollector = null,
 		ILegacyUrlMapper? legacyUrlMapper = null,
 		IPositionalNavigation? positionalNavigation = null
 	)
 	{
+		_markdownExporters = markdownExporters ?? [];
 		_documentationFileOutputProvider = documentationFileOutputProvider;
 		_conversionCollector = conversionCollector;
 		_writeFileSystem = docSet.Context.WriteFileSystem;
@@ -100,7 +104,7 @@ public async Task<GenerationResult> GenerateAll(Cancel ctx)
 
 		var generationState = Context.SkipDocumentationState ? null : GetPreviousGenerationState();
 
-		// clear output directory if force is true but never for assembler builds since these build multiple times to the output.
+		// clear the output directory if force is true but never for assembler builds since these build multiple times to the output.
 		if (Context is { AssemblerBuild: false, Force: true }
 			// clear the output directory if force is false but generation state is null, except for assembler builds.
 			|| (Context is { AssemblerBuild: false, Force: false } && generationState == null))
@@ -209,7 +213,7 @@ private async Task ExtractEmbeddedStaticResources(Cancel ctx)
 		}
 	}
 
-	private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile file, DateTimeOffset outputSeenChanges, Cancel token)
+	private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile file, DateTimeOffset outputSeenChanges, Cancel ctx)
 	{
 		if (!Context.Force)
 		{
@@ -220,10 +224,27 @@ private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile
 		}
 
 		_logger.LogTrace("--> {FileFullPath}", file.SourceFile.FullName);
-		//TODO send file to OutputFile() so we can validate its scope is defined in navigation.yml
 		var outputFile = OutputFile(file.RelativePath);
 		if (outputFile is not null)
-			await _documentationFileExporter.ProcessFile(Context, file, outputFile, HtmlWriter, _conversionCollector, token);
+		{
+			var context = new ProcessingFileContext
+			{
+				BuildContext = Context,
+				OutputFile = outputFile,
+				ConversionCollector = _conversionCollector,
+				File = file,
+				HtmlWriter = HtmlWriter
+			};
+			await _documentationFileExporter.ProcessFile(context, ctx);
+			if (file is MarkdownFile markdown)
+			{
+				foreach (var exporter in _markdownExporters)
+				{
+					var document = context.MarkdownDocument ??= await markdown.ParseFullAsync(ctx);
+					_ = await exporter.ExportAsync(new MarkdownExportContext { Document = document, File = markdown }, ctx);
+				}
+			}
+		}
 	}
 
 	private IFileInfo? OutputFile(string relativePath)