Synonyms as code

config/synonyms.yml

@@ -0,0 +1,24 @@
+synonyms:
+  - ".net,.NET,c#,csharp,dotnet,net,NET,DOTNET"
+  - "esql,es|ql"
+  - "motlp,managed otlp,Managed OTLP"
+  - "s3,aws s3,amazon s3"
+  - "es,elasticsearch"
+  - "Elastic Learned Sparse EncodeR,elser"
+  - "ccs,cross cluster search,cross-cluster search,spaghetti"
+  - "apm,application performance monitoring"
+  - "ecctl,Elastic Cloud Control"
+  - "opentelemetry,otel"
+  - "eck,Elastic Cloud on Kubernetes"
+  - "ece,Elastic Cloud Enterprise"
+  - "ELv2,Elastic License v2"
+  - "kql,Kibana Query Language"
+  - "ccr,cross-cluster replication,cross cluster replication"
+  - "ESaaS,Elastic Stack as a service"
+  - "knn,k-Nearest Neighbors"
+  - "ech,Elastic Cloud Hosted,ess"
+  - "aws,amazon"
+  - "ilm,index lifecycle management"
+  - "javascript,js,node,nodejs,node.js"
+  - "edot,elastic distribution of opentelemetry"
+  - "k8s,kubernetes"

docs-builder.sln

@@ -122,6 +122,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "config", "config", "{6FAB56
 		config\legacy-url-mappings.yml = config\legacy-url-mappings.yml
 		config\navigation.yml = config\navigation.yml
 		config\products.yml = config\products.yml
+		config\synonyms.yml = config\synonyms.yml
 	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests-integration", "tests-integration", "{BCAD38D5-6C83-46E2-8398-4BE463931098}"

src/Elastic.Documentation.Configuration/BuildContext.cs

@@ -8,6 +8,7 @@
 using Elastic.Documentation.Configuration.Builder;
 using Elastic.Documentation.Configuration.LegacyUrlMappings;
 using Elastic.Documentation.Configuration.Products;
+using Elastic.Documentation.Configuration.Synonyms;
 using Elastic.Documentation.Configuration.Versions;
 using Elastic.Documentation.Diagnostics;
 
@@ -34,6 +35,7 @@ public record BuildContext : IDocumentationSetContext, IDocumentationConfigurati
 
 	public ProductsConfiguration ProductsConfiguration { get; }
 	public LegacyUrlMappingConfiguration LegacyUrlMappings { get; }
+	public SynonymsConfiguration SynonymsConfiguration { get; }
 
 	public IFileInfo ConfigurationPath { get; }
 
@@ -85,6 +87,7 @@ public BuildContext(
 		ReadFileSystem = readFileSystem;
 		WriteFileSystem = writeFileSystem;
 		AvailableExporters = availableExporters;
+		SynonymsConfiguration = configurationContext.SynonymsConfiguration;
 		VersionsConfiguration = configurationContext.VersionsConfiguration;
 		ConfigurationFileProvider = configurationContext.ConfigurationFileProvider;
 		ProductsConfiguration = configurationContext.ProductsConfiguration;

src/Elastic.Documentation.Configuration/ConfigurationFileProvider.cs

@@ -49,7 +49,7 @@ public ConfigurationFileProvider(
 			ConfigurationSource = source;
 		else
 		{
-			string[] spotChecks = ["navigation.yml", "versions.yml", "products.yml", "assembler.yml"];
+			string[] spotChecks = ["navigation.yml", "versions.yml", "products.yml", "assembler.yml", "synonyms.yml"];
 			var defaultSource =
 				fileSystem.Directory.Exists(LocalConfigurationDirectory)
 					&& spotChecks.All(f => fileSystem.File.Exists(Path.Combine(LocalConfigurationDirectory, f)))
@@ -92,6 +92,7 @@ public ConfigurationFileProvider(
 		AssemblerFile = CreateTemporaryConfigurationFile("assembler.yml");
 		NavigationFile = CreateTemporaryConfigurationFile("navigation.yml");
 		LegacyUrlMappingsFile = CreateTemporaryConfigurationFile("legacy-url-mappings.yml");
+		SynonymsFile = CreateTemporaryConfigurationFile("synonyms.yml");
 	}
 
 	public bool SkipPrivateRepositories { get; }
@@ -108,6 +109,7 @@ public ConfigurationFileProvider(
 
 	public IFileInfo LegacyUrlMappingsFile { get; }
 
+	public IFileInfo SynonymsFile { get; }
 	public IFileInfo CreateNavigationFile(AssemblyConfiguration configuration)
 	{
 		var privateRepositories = configuration.PrivateRepositories;

src/Elastic.Documentation.Configuration/Elastic.Documentation.Configuration.csproj

@@ -24,5 +24,6 @@
     <EmbeddedResource Include="$(SolutionRoot)\config\assembler.yml" />
     <EmbeddedResource Include="$(SolutionRoot)\config\navigation.yml" />
     <EmbeddedResource Include="$(SolutionRoot)\config\legacy-url-mappings.yml" />
+    <EmbeddedResource Include="$(SolutionRoot)\config\synonyms.yml" />
   </ItemGroup>
 </Project>

src/Elastic.Documentation.Configuration/IDocumentationConfigurationContext.cs

@@ -4,6 +4,7 @@
 
 using Elastic.Documentation.Configuration.LegacyUrlMappings;
 using Elastic.Documentation.Configuration.Products;
+using Elastic.Documentation.Configuration.Synonyms;
 using Elastic.Documentation.Configuration.Versions;
 
 namespace Elastic.Documentation.Configuration;
@@ -15,6 +16,8 @@ public interface IConfigurationContext
 	DocumentationEndpoints Endpoints { get; }
 	ProductsConfiguration ProductsConfiguration { get; }
 	LegacyUrlMappingConfiguration LegacyUrlMappings { get; }
+	SynonymsConfiguration SynonymsConfiguration { get; }
+
 }
 
 /// Used only to seed <see cref="IConfigurationContext"/> in DI, you primarily want to depend on <see cref="IDocumentationConfigurationContext"/>
@@ -35,6 +38,8 @@ public class ConfigurationContext : IConfigurationContext
 	/// <inheritdoc />
 	public required LegacyUrlMappingConfiguration LegacyUrlMappings { get; init; }
 
+	/// <inheritdoc />
+	public required SynonymsConfiguration SynonymsConfiguration { get; init; }
 }
 
 public interface IDocumentationConfigurationContext : IDocumentationContext, IConfigurationContext;

src/Elastic.Documentation.Configuration/Serialization/YamlStaticContext.cs

@@ -5,6 +5,7 @@
 using Elastic.Documentation.Configuration.Assembler;
 using Elastic.Documentation.Configuration.LegacyUrlMappings;
 using Elastic.Documentation.Configuration.Products;
+using Elastic.Documentation.Configuration.Synonyms;
 using Elastic.Documentation.Configuration.Versions;
 using YamlDotNet.Serialization;
 
@@ -23,4 +24,5 @@ namespace Elastic.Documentation.Configuration.Serialization;
 [YamlSerializable(typeof(ProductDto))]
 [YamlSerializable(typeof(LegacyUrlMappingDto))]
 [YamlSerializable(typeof(LegacyUrlMappingConfigDto))]
+[YamlSerializable(typeof(SynonymsConfigDto))] // Add this line
 public partial class YamlStaticContext;

src/Elastic.Documentation.Configuration/Synonyms/SynonymsConfiguration.cs

@@ -0,0 +1,25 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+namespace Elastic.Documentation.Configuration.Synonyms;
+
+public record SynonymsConfiguration
+{
+	public required IReadOnlyCollection<string> Synonyms { get; init; }
+}
+
+internal sealed record SynonymsConfigDto
+{
+	public List<string> Synonyms { get; set; } = [];
+}
+
+public static class SynonymsConfigurationExtensions
+{
+	public static SynonymsConfiguration CreateSynonymsConfiguration(this ConfigurationFileProvider provider)
+	{
+		var synonymsFile = provider.SynonymsFile;
+		var synonymsDto = ConfigurationFileProvider.Deserializer.Deserialize<SynonymsConfigDto>(synonymsFile.OpenText());
+		return new SynonymsConfiguration { Synonyms = synonymsDto.Synonyms };
+	}
+}

src/Elastic.Documentation.ServiceDefaults/AppDefaultsExtensions.cs

@@ -5,6 +5,7 @@
 using Elastic.Documentation.Configuration;
 using Elastic.Documentation.Configuration.LegacyUrlMappings;
 using Elastic.Documentation.Configuration.Products;
+using Elastic.Documentation.Configuration.Synonyms;
 using Elastic.Documentation.Configuration.Versions;
 using Elastic.Documentation.ServiceDefaults.Logging;
 using Microsoft.Extensions.DependencyInjection;
@@ -34,9 +35,11 @@ public static TBuilder AddDocumentationServiceDefaults<TBuilder>(this TBuilder b
 			{
 				var versionConfiguration = p.CreateVersionConfiguration();
 				var products = p.CreateProducts(versionConfiguration);
+				var synonyms = p.CreateSynonymsConfiguration();
 				_ = s.AddSingleton(p.CreateLegacyUrlMappings(products));
 				_ = s.AddSingleton(products);
 				_ = s.AddSingleton(versionConfiguration);
+				_ = s.AddSingleton(synonyms);
 				configure?.Invoke(s, p);
 			});
 		_ = builder.Services.AddElasticDocumentationLogging(globalArgs.LogLevel);

src/Elastic.Documentation/Serialization/SourceGenerationContext.cs

@@ -28,4 +28,5 @@ namespace Elastic.Documentation.Serialization;
 [JsonSerializable(typeof(Applicability))]
 [JsonSerializable(typeof(ProductLifecycle))]
 [JsonSerializable(typeof(SemVersion))]
+[JsonSerializable(typeof(string[]))]
 public sealed partial class SourceGenerationContext : JsonSerializerContext;

src/Elastic.Markdown/Exporters/Elasticsearch/ElasticsearchExporter.cs

@@ -33,7 +33,7 @@ DistributedTransport transport
 				{ "batch_index_date", d.BatchIndexDate.ToString("o") }
 			}),
 		GetMapping = () => CreateMapping(null),
-		GetMappingSettings = CreateMappingSetting,
+		GetMappingSettings = () => CreateMappingSetting("docs"),
 		IndexFormat =
 			$"{endpoint.IndexNamePrefix.Replace("semantic", "lexical").ToLowerInvariant()}-{indexNamespace.ToLowerInvariant()}-{{0:yyyy.MM.dd.HHmmss}}",
 		ActiveSearchAlias = $"{endpoint.IndexNamePrefix.Replace("semantic", "lexical").ToLowerInvariant()}-{indexNamespace.ToLowerInvariant()}"
@@ -51,7 +51,7 @@ DistributedTransport transport
 	{
 		BulkOperationIdLookup = d => d.Url,
 		GetMapping = (inferenceId, _) => CreateMapping(inferenceId),
-		GetMappingSettings = (_, _) => CreateMappingSetting(),
+		GetMappingSettings = (_, _) => CreateMappingSetting("docs"),
 		IndexFormat = $"{endpoint.IndexNamePrefix.ToLowerInvariant()}-{indexNamespace.ToLowerInvariant()}-{{0:yyyy.MM.dd.HHmmss}}",
 		ActiveSearchAlias = $"{endpoint.IndexNamePrefix}-{indexNamespace.ToLowerInvariant()}",
 		IndexNumThreads = endpoint.IndexNumThreads,
@@ -139,9 +139,9 @@ public async ValueTask<bool> TryWrite(DocumentationDocument document, Cancel ctx
 	}
 
 
-	protected static string CreateMappingSetting() =>
+	protected static string CreateMappingSetting(string synonymSetName) =>
 		// language=json
-		"""
+		$$"""
 		{
 		  "analysis": {
 		    "analyzer": {
@@ -163,10 +163,9 @@ protected static string CreateMappingSetting() =>
 		    },
 		    "filter": {
 		      "synonyms_filter": {
-		        "type": "synonym",
-		        "synonyms_set": "docs",
-		        "updateable": true
-		      },
+				  "type": "synonym_graph",
+				  "synonyms_set": "{{synonymSetName}}"
+				  },
 		      "english_stop": {
 		        "type": "stop",
 		        "stopwords": "_english_"

src/Elastic.Markdown/Exporters/Elasticsearch/ElasticsearchMarkdownExporter.cs

@@ -4,7 +4,10 @@
 
 using System.IO.Abstractions;
 using Elastic.Documentation.AppliesTo;
+using System.Text.Json;
+using System.Text.Json.Serialization;
 using Elastic.Documentation.Configuration;
+using Elastic.Documentation.Configuration.Synonyms;
 using Elastic.Documentation.Diagnostics;
 using Elastic.Documentation.Search;
 using Elastic.Ingest.Elasticsearch;
@@ -22,6 +25,16 @@ namespace Elastic.Markdown.Exporters.Elasticsearch;
 [EnumExtensions]
 public enum IngestStrategy { Reindex, Multiplex }
 
+internal sealed record SynonymSetRequest
+{
+	[JsonPropertyName("synonyms")]
+	public required string[] Synonyms { get; init; }
+}
+
+[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.SnakeCaseLower)]
+[JsonSerializable(typeof(SynonymSetRequest))]
+internal sealed partial class SynonymSerializerContext : JsonSerializerContext { };
+
 public class ElasticsearchMarkdownExporter : IMarkdownExporter, IDisposable
 {
 	private readonly IDiagnosticsCollector _collector;
@@ -37,11 +50,14 @@ public class ElasticsearchMarkdownExporter : IMarkdownExporter, IDisposable
 	private string _currentLexicalHash = string.Empty;
 	private string _currentSemanticHash = string.Empty;
 
+	private readonly IReadOnlyCollection<string> _synonyms;
+
 	public ElasticsearchMarkdownExporter(
 		ILoggerFactory logFactory,
 		IDiagnosticsCollector collector,
 		DocumentationEndpoints endpoints,
-		string indexNamespace
+		string indexNamespace,
+		SynonymsConfiguration synonyms
 	)
 	{
 		_collector = collector;
@@ -75,7 +91,7 @@ string indexNamespace
 		};
 
 		_transport = new DistributedTransport(configuration);
-
+		_synonyms = synonyms.Synonyms;
 		_lexicalChannel = new ElasticsearchLexicalExporter(logFactory, collector, es, indexNamespace, _transport);
 		_semanticChannel = new ElasticsearchSemanticExporter(logFactory, collector, es, indexNamespace, _transport);
 	}
@@ -86,6 +102,7 @@ public async ValueTask StartAsync(Cancel ctx = default)
 		_currentLexicalHash = await _lexicalChannel.Channel.GetIndexTemplateHashAsync(ctx) ?? string.Empty;
 		_currentSemanticHash = await _semanticChannel.Channel.GetIndexTemplateHashAsync(ctx) ?? string.Empty;
 
+		await PublishSynonymsAsync("docs", ctx);
 		_ = await _lexicalChannel.Channel.BootstrapElasticsearchAsync(BootstrapMethod.Failure, null, ctx);
 
 		// if the previous hash does not match the current hash, we know already we want to multiplex to a new index
@@ -132,6 +149,25 @@ public async ValueTask StartAsync(Cancel ctx = default)
 		async ValueTask<bool> IndexExists(string name) => (await _transport.HeadAsync(name, ctx)).ApiCallDetails.HasSuccessfulStatusCode;
 	}
 
+	private async Task PublishSynonymsAsync(string setName, CancellationToken ctx)
+	{
+		_logger.LogInformation("Publishing synonym set '{SetName}' to Elasticsearch", setName);
+
+		var requestBody = new SynonymSetRequest { Synonyms = _synonyms.ToArray() };
+		var json = JsonSerializer.Serialize(requestBody, SynonymSerializerContext.Default.SynonymSetRequest);
+
+		var response = await _transport.PutAsync<StringResponse>($"_synonyms/{setName}", PostData.String(json), ctx);
+
+		if (!response.ApiCallDetails.HasSuccessfulStatusCode)
+		{
+			_collector.EmitGlobalError($"Failed to publish synonym set '{setName}'. Reason: {response.ApiCallDetails.OriginalException?.Message ?? response.ToString()}");
+		}
+		else
+		{
+			_logger.LogInformation("Successfully published synonym set '{SetName}'.", setName);
+		}
+	}
+
 	private async ValueTask<long> CountAsync(string index, string body, Cancel ctx = default)
 	{
 		var countResponse = await _transport.PostAsync<DynamicResponse>($"/{index}/_count", PostData.String(body), ctx);

src/Elastic.Markdown/Exporters/ExporterExtensions.cs

@@ -25,7 +25,7 @@ string indexNamespace
 		if (exportOptions.Contains(Exporter.Configuration))
 			markdownExporters.Add(new ConfigurationExporter(logFactory, context.ConfigurationFileProvider, context));
 		if (exportOptions.Contains(Exporter.Elasticsearch))
-			markdownExporters.Add(new ElasticsearchMarkdownExporter(logFactory, context.Collector, context.Endpoints, indexNamespace));
+			markdownExporters.Add(new ElasticsearchMarkdownExporter(logFactory, context.Collector, context.Endpoints, indexNamespace, context.SynonymsConfiguration));
 		return markdownExporters;
 	}
 }

src/services/Elastic.Documentation.Assembler/AssembleContext.cs

@@ -7,6 +7,7 @@
 using Elastic.Documentation.Configuration.Assembler;
 using Elastic.Documentation.Configuration.LegacyUrlMappings;
 using Elastic.Documentation.Configuration.Products;
+using Elastic.Documentation.Configuration.Synonyms;
 using Elastic.Documentation.Configuration.Versions;
 using Elastic.Documentation.Diagnostics;
 
@@ -31,6 +32,7 @@ public class AssembleContext : IDocumentationConfigurationContext
 
 	public ProductsConfiguration ProductsConfiguration { get; }
 	public LegacyUrlMappingConfiguration LegacyUrlMappings { get; }
+	public SynonymsConfiguration SynonymsConfiguration { get; }
 
 	public IDirectoryInfo CheckoutDirectory { get; }
 
@@ -56,6 +58,7 @@ public AssembleContext(
 		Configuration = configuration;
 		ConfigurationFileProvider = configurationContext.ConfigurationFileProvider;
 		VersionsConfiguration = configurationContext.VersionsConfiguration;
+		SynonymsConfiguration = configurationContext.SynonymsConfiguration;
 		Endpoints = configurationContext.Endpoints;
 		ProductsConfiguration = configurationContext.ProductsConfiguration;
 		LegacyUrlMappings = configurationContext.LegacyUrlMappings;

src/tooling/Elastic.Documentation.Tooling/DocumentationTooling.cs

@@ -9,6 +9,7 @@
 using Elastic.Documentation.Configuration;
 using Elastic.Documentation.Configuration.LegacyUrlMappings;
 using Elastic.Documentation.Configuration.Products;
+using Elastic.Documentation.Configuration.Synonyms;
 using Elastic.Documentation.Configuration.Versions;
 using Elastic.Documentation.Diagnostics;
 using Elastic.Documentation.ServiceDefaults;
@@ -76,13 +77,15 @@ public static TBuilder AddDocumentationToolingDefaults<TBuilder>(this TBuilder b
 				var versionsConfiguration = sp.GetRequiredService<VersionsConfiguration>();
 				var products = sp.GetRequiredService<ProductsConfiguration>();
 				var legacyUrlMappings = sp.GetRequiredService<LegacyUrlMappingConfiguration>();
+				var synonyms = sp.GetRequiredService<SynonymsConfiguration>(); // Add this line
 				return new ConfigurationContext
 				{
 					ConfigurationFileProvider = configurationFileProvider,
 					VersionsConfiguration = versionsConfiguration,
 					Endpoints = endpoints,
 					ProductsConfiguration = products,
-					LegacyUrlMappings = legacyUrlMappings
+					LegacyUrlMappings = legacyUrlMappings,
+					SynonymsConfiguration = synonyms
 				};
 			});