Refactor Elasticsearch exporter mappings, optimize ApplicableTo handling, and improve index creation logic

Author: Mpdreamz

src/Elastic.Documentation/AppliesTo/ApplicableToJsonConverter.cs

@@ -167,9 +167,7 @@ public override void Write(Utf8JsonWriter writer, ApplicableTo value, JsonSerial
 
 		// Stack
 		if (value.Stack != null)
-		{
 			WriteApplicabilityEntries(writer, "stack", "stack", value.Stack);
-		}
 
 		// Deployment
 		if (value.Deployment != null)
@@ -197,9 +195,7 @@ public override void Write(Utf8JsonWriter writer, ApplicableTo value, JsonSerial
 
 		// Product (simple)
 		if (value.Product != null)
-		{
 			WriteApplicabilityEntries(writer, "product", "product", value.Product);
-		}
 
 		// ProductApplicability (specific products)
 		if (value.ProductApplicability != null)
@@ -210,8 +206,7 @@ public override void Write(Utf8JsonWriter writer, ApplicableTo value, JsonSerial
 				var yamlAlias = property.GetCustomAttribute<YamlMemberAttribute>()?.Alias;
 				if (yamlAlias != null)
 				{
-					var propertyValue = property.GetValue(value.ProductApplicability) as AppliesCollection;
-					if (propertyValue != null)
+					if (property.GetValue(value.ProductApplicability) is AppliesCollection propertyValue)
 						WriteApplicabilityEntries(writer, "product", yamlAlias, propertyValue);
 				}
 			}
@@ -220,22 +215,19 @@ public override void Write(Utf8JsonWriter writer, ApplicableTo value, JsonSerial
 		writer.WriteEndArray();
 	}
 
-	private static ProductLifecycle ParseLifecycle(string lifecycleStr)
+	private static ProductLifecycle ParseLifecycle(string lifecycleStr) => lifecycleStr.ToLowerInvariant() switch
 	{
-		return lifecycleStr.ToLowerInvariant() switch
-		{
-			"preview" => ProductLifecycle.TechnicalPreview,
-			"beta" => ProductLifecycle.Beta,
-			"ga" => ProductLifecycle.GenerallyAvailable,
-			"deprecated" => ProductLifecycle.Deprecated,
-			"removed" => ProductLifecycle.Removed,
-			"unavailable" => ProductLifecycle.Unavailable,
-			"development" => ProductLifecycle.Development,
-			"planned" => ProductLifecycle.Planned,
-			"discontinued" => ProductLifecycle.Discontinued,
-			_ => ProductLifecycle.GenerallyAvailable
-		};
-	}
+		"preview" => ProductLifecycle.TechnicalPreview,
+		"beta" => ProductLifecycle.Beta,
+		"ga" => ProductLifecycle.GenerallyAvailable,
+		"deprecated" => ProductLifecycle.Deprecated,
+		"removed" => ProductLifecycle.Removed,
+		"unavailable" => ProductLifecycle.Unavailable,
+		"development" => ProductLifecycle.Development,
+		"planned" => ProductLifecycle.Planned,
+		"discontinued" => ProductLifecycle.Discontinued,
+		_ => ProductLifecycle.GenerallyAvailable
+	};
 
 	private static void WriteApplicabilityEntries(Utf8JsonWriter writer, string type, string subType, AppliesCollection collection)
 	{
@@ -261,7 +253,7 @@ private static void WriteApplicabilityEntries(Utf8JsonWriter writer, string type
 			};
 			writer.WriteString("lifecycle", lifecycleName);
 
-			// Write version
+			// Write the version
 			if (applicability.Version is not null)
 				writer.WriteString("version", applicability.Version.ToString());
 

src/Elastic.Markdown/Exporters/Elasticsearch/ElasticsearchExporter.cs

@@ -193,15 +193,15 @@ protected static string CreateMapping(string? inferenceId) =>
 		          "prefix": { "type": "text", "analyzer" : "hierarchy_analyzer" }
 		        }
 		      },
-		      "applies" : {
-		        "type" : "nested"
+		      "applies_to" : {
+		        "type" : "nested",
 		        "properties" : {
 		          "type" : { "type" : "keyword" },
 		          "sub-type" : { "type" : "keyword" },
 		          "lifecycle" : { "type" : "keyword" },
-		          "version" : { "type" : "version" },
-		        }
-		      },
+		          "version" : { "type" : "version" }
+		         }
+		       },
 		      "hash" : { "type" : "keyword" },
 		      "title": {
 		        "type": "text",

src/Elastic.Markdown/Exporters/Elasticsearch/ElasticsearchMarkdownExporter.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information
 
 using System.IO.Abstractions;
+using Elastic.Documentation.AppliesTo;
 using Elastic.Documentation.Configuration;
 using Elastic.Documentation.Diagnostics;
 using Elastic.Documentation.Search;
@@ -82,17 +83,16 @@ public async ValueTask StartAsync(Cancel ctx = default)
 	{
 		_ = await _lexicalChannel.Channel.BootstrapElasticsearchAsync(BootstrapMethod.Failure, null, ctx);
 
-		var semanticIndex = _semanticChannel.Channel.IndexName;
 		var semanticWriteAlias = string.Format(_semanticChannel.Channel.Options.IndexFormat, "latest");
-		var semanticIndexHead = await _transport.HeadAsync(semanticWriteAlias, ctx);
-		if (!semanticIndexHead.ApiCallDetails.HasSuccessfulStatusCode)
+		var semanticIndexAvailable = await _transport.HeadAsync(semanticWriteAlias, ctx);
+		_ = await _semanticChannel.Channel.BootstrapElasticsearchAsync(BootstrapMethod.Failure, null, ctx);
+		var semanticIndex = _semanticChannel.Channel.IndexName;
+		if (!semanticIndexAvailable.ApiCallDetails.HasSuccessfulStatusCode)
 		{
-			_logger.LogInformation("No semantic index exists yet, creating index {Index} for semantic search", semanticIndex);
-			_ = await _semanticChannel.Channel.BootstrapElasticsearchAsync(BootstrapMethod.Failure, null, ctx);
+			_logger.LogInformation("No semantic index existed yet, creating index {Index} for semantic search", semanticIndex);
 			var semanticIndexPut = await _transport.PutAsync<StringResponse>(semanticIndex, PostData.String("{}"), ctx);
 			if (!semanticIndexPut.ApiCallDetails.HasSuccessfulStatusCode)
 				throw new Exception($"Failed to create index {semanticIndex}: {semanticIndexPut}");
-			_ = await _semanticChannel.Channel.ApplyAliasesAsync(ctx);
 			if (!_endpoint.ForceReindex)
 			{
 				_indexStrategy = IngestStrategy.Multiplex;
@@ -152,12 +152,13 @@ public async ValueTask StopAsync(Cancel ctx = default)
 				throw new Exception($"Failed to create index {semanticIndex}: {semanticIndexPut}");
 			_ = await _semanticChannel.Channel.ApplyAliasesAsync(ctx);
 		}
+		var destinationIndex = _semanticChannel.Channel.IndexName;
 
-		_logger.LogInformation("_reindex updates: '{SourceIndex}' => '{DestinationIndex}'", lexicalWriteAlias, semanticWriteAlias);
+		_logger.LogInformation("_reindex updates: '{SourceIndex}' => '{DestinationIndex}'", lexicalWriteAlias, destinationIndex);
 		var request = PostData.String(@"
 		{
 			""dest"": {
-				""index"": """ + semanticWriteAlias + @"""
+				""index"": """ + destinationIndex + @"""
 			},
 			""source"": {
 				""index"": """ + lexicalWriteAlias + @""",
@@ -171,13 +172,13 @@ public async ValueTask StopAsync(Cancel ctx = default)
 				}
 			}
 		}");
-		await DoReindex(request, lexicalWriteAlias, semanticWriteAlias, "updates", ctx);
+		await DoReindex(request, lexicalWriteAlias, destinationIndex, "updates", ctx);
 
-		_logger.LogInformation("_reindex deletions: '{SourceIndex}' => '{DestinationIndex}'", lexicalWriteAlias, semanticWriteAlias);
+		_logger.LogInformation("_reindex deletions: '{SourceIndex}' => '{DestinationIndex}'", lexicalWriteAlias, destinationIndex);
 		request = PostData.String(@"
 		{
 			""dest"": {
-				""index"": """ + semanticWriteAlias + @"""
+				""index"": """ + destinationIndex + @"""
 			},
 			""script"": {
 				""source"": ""ctx.op = \""delete\""""
@@ -194,7 +195,7 @@ public async ValueTask StopAsync(Cancel ctx = default)
 				}
 			}
 		}");
-		await DoReindex(request, lexicalWriteAlias, semanticWriteAlias, "deletions", ctx);
+		await DoReindex(request, lexicalWriteAlias, destinationIndex, "deletions", ctx);
 
 		await DoDeleteByQuery(lexicalWriteAlias, ctx);
 
@@ -336,6 +337,10 @@ public async ValueTask<bool> ExportAsync(MarkdownExportFileContext fileContext,
 			? body[..Math.Min(body.Length, 400)] + " " + string.Join(" \n- ", headings)
 			: string.Empty;
 
+		// this is temporary until https://github.com/elastic/docs-builder/pull/2070 lands
+		// this PR will add a service for us to resolve to a versioning scheme.
+		var appliesTo = fileContext.SourceFile.YamlFrontMatter?.AppliesTo ?? ApplicableTo.Default;
+
 		var doc = new DocumentationDocument
 		{
 			Url = url,
@@ -344,7 +349,7 @@ public async ValueTask<bool> ExportAsync(MarkdownExportFileContext fileContext,
 			StrippedBody = body.StripMarkdown(),
 			Description = fileContext.SourceFile.YamlFrontMatter?.Description,
 			Abstract = @abstract,
-			Applies = fileContext.SourceFile.YamlFrontMatter?.AppliesTo,
+			Applies = appliesTo,
 			UrlSegmentCount = url.Split('/', StringSplitOptions.RemoveEmptyEntries).Length,
 			Parents = navigation.GetParentsOfMarkdownFile(file).Select(i => new ParentDocument
 			{
@@ -357,7 +362,7 @@ public async ValueTask<bool> ExportAsync(MarkdownExportFileContext fileContext,
 		var semanticHash = _semanticChannel.Channel.ChannelHash;
 		var lexicalHash = _lexicalChannel.Channel.ChannelHash;
 		var hash = HashedBulkUpdate.CreateHash(semanticHash, lexicalHash,
-			doc.Url, doc.Body ?? string.Empty, string.Join(",", doc.Headings.OrderBy(h => h))
+			doc.Url, doc.Body ?? string.Empty, string.Join(",", doc.Headings.OrderBy(h => h)), doc.Url
 		);
 		doc.Hash = hash;
 		doc.LastUpdated = _batchIndexDate;