Specialized index command aliases with more options (#1934)

* Add specialized index commands

* Fix --version and --help being too noisy

* Ensure exporters are start and stopped during isolated builds

* Improve export logging a tad

* Ensure assemble build shows no hints but we now expose --show-hints to enable it

* build failures

Author: Mpdreamz

aspire/AppHost.cs

@@ -13,7 +13,7 @@
 // ReSharper disable NotAccessedVariable
 
 var logLevel = LogLevel.Information;
-GlobalCommandLine.Process(ref args, ref logLevel, out var skipPrivateRepositories);
+GlobalCommandLine.Process(ref args, ref logLevel, out var skipPrivateRepositories, out _);
 var globalArguments = new List<string>();
 if (skipPrivateRepositories)
 	globalArguments.Add("--skip-private-repositories");

src/Elastic.Documentation.Configuration/DocumentationEndpoints.cs

@@ -2,19 +2,45 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System.Security.Cryptography.X509Certificates;
+
 namespace Elastic.Documentation.Configuration;
 
-public record DocumentationEndpoints
+public class DocumentationEndpoints
 {
 	public required ElasticsearchEndpoint Elasticsearch { get; init; }
 }
 
-public record ElasticsearchEndpoint
+public class ElasticsearchEndpoint
 {
-	public static ElasticsearchEndpoint Default { get; } = new ElasticsearchEndpoint { Uri = new Uri("https://localhost:9200") };
+	public static ElasticsearchEndpoint Default { get; } = new() { Uri = new Uri("https://localhost:9200") };
+
+	public required Uri Uri { get; set; }
+	public string? Username { get; set; }
+	public string? Password { get; set; }
+	public string? ApiKey { get; set; }
+
+	// inference options
+	public int SearchNumThreads { get; set; } = 8;
+	public int IndexNumThreads { get; set; } = 8;
+
+	// index options
+	public string IndexNamePrefix { get; set; } = "semantic-docs";
+
+	// channel buffer options
+	public int BufferSize { get; set; } = 100;
+	public int MaxRetries { get; set; } = 3;
+
+
+	// connection options
+	public bool DebugMode { get; set; }
+	public string? CertificateFingerprint { get; set; }
+	public string? ProxyAddress { get; set; }
+	public string? ProxyPassword { get; set; }
+	public string? ProxyUsername { get; set; }
 
-	public required Uri Uri { get; init; }
-	public string? Username { get; init; }
-	public string? Password { get; init; }
-	public string? ApiKey { get; init; }
+	public bool DisableSslVerification { get; set; }
+	public X509Certificate? Certificate { get; set; }
+	public bool CertificateIsNotRoot { get; set; }
+	public int? BootstrapTimeout { get; set; }
 }

src/Elastic.Documentation.ServiceDefaults/AppDefaultsExtensions.cs

@@ -13,6 +13,8 @@
 
 namespace Elastic.Documentation.ServiceDefaults;
 
+public record CliInvocation(bool IsHelpOrVersion);
+
 public static class AppDefaultsExtensions
 {
 	public static TBuilder AddDocumentationServiceDefaults<TBuilder>(this TBuilder builder) where TBuilder : IHostApplicationBuilder
@@ -26,7 +28,7 @@ public static TBuilder AddDocumentationServiceDefaults<TBuilder>(this TBuilder b
 	public static TBuilder AddDocumentationServiceDefaults<TBuilder>(this TBuilder builder, ref string[] args, LogLevel? defaultLogLevel = null, Action<IServiceCollection, ConfigurationFileProvider>? configure = null) where TBuilder : IHostApplicationBuilder
 	{
 		var logLevel = defaultLogLevel ?? LogLevel.Information;
-		GlobalCommandLine.Process(ref args, ref logLevel, out var skipPrivateRepositories);
+		GlobalCommandLine.Process(ref args, ref logLevel, out var skipPrivateRepositories, out var isHelpOrVersion);
 
 		var services = builder.Services;
 		_ = services
@@ -36,6 +38,7 @@ public static TBuilder AddDocumentationServiceDefaults<TBuilder>(this TBuilder b
 				configure?.Invoke(s, p);
 			});
 		_ = builder.Services.AddElasticDocumentationLogging(logLevel);
+		_ = services.AddSingleton(new CliInvocation(isHelpOrVersion));
 
 		return builder.AddServiceDefaults();
 	}

src/Elastic.Documentation/Diagnostics/DiagnosticsCollector.cs

@@ -29,7 +29,7 @@ public class DiagnosticsCollector(IReadOnlyCollection<IDiagnosticsOutput> output
 
 	public ConcurrentBag<string> CrossLinks { get; } = [];
 
-	public bool NoHints { get; init; }
+	public bool NoHints { get; set; }
 
 	public DiagnosticsCollector StartAsync(Cancel ctx)
 	{

src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs

@@ -14,6 +14,8 @@ public interface IDiagnosticsCollector : IAsyncDisposable, IHostedService
 	int Errors { get; }
 	int Hints { get; }
 
+	bool NoHints { get; set; }
+
 	DiagnosticsChannel Channel { get; }
 	ConcurrentBag<string> CrossLinks { get; }
 	HashSet<string> OffendingFiles { get; }

src/Elastic.Documentation/Exporter.cs

@@ -11,7 +11,7 @@ public enum Exporter
 	Html,
 	LLMText,
 	Elasticsearch,
-	SemanticElasticsearch,
+	ElasticsearchNoSemantic,
 	Configuration,
 	DocumentationState,
 	LinkMetadata,

src/Elastic.Documentation/GlobalCommandLine.cs

@@ -8,9 +8,15 @@ namespace Elastic.Documentation;
 
 public static class GlobalCommandLine
 {
-	public static void Process(ref string[] args, ref LogLevel defaultLogLevel, out bool skipPrivateRepositories)
+	public static void Process(
+		ref string[] args,
+		ref LogLevel defaultLogLevel,
+		out bool skipPrivateRepositories,
+		out bool isHelpOrVersion
+	)
 	{
 		skipPrivateRepositories = false;
+		isHelpOrVersion = false;
 		var newArgs = new List<string>();
 		for (var i = 0; i < args.Length; i++)
 		{
@@ -22,8 +28,11 @@ public static void Process(ref string[] args, ref LogLevel defaultLogLevel, out
 			}
 			else if (args[i] == "--skip-private-repositories")
 				skipPrivateRepositories = true;
-			else if (args[i] == "--inject")
-				skipPrivateRepositories = true;
+			else if (args[i] is "--help" or "--version")
+			{
+				isHelpOrVersion = true;
+				newArgs.Add(args[i]);
+			}
 			else
 				newArgs.Add(args[i]);
 		}

src/Elastic.Markdown/Exporters/ElasticsearchMarkdownExporter.cs

@@ -5,7 +5,6 @@
 using System.IO.Abstractions;
 using Elastic.Channels;
 using Elastic.Documentation.Configuration;
-using Elastic.Documentation.Configuration.Assembler;
 using Elastic.Documentation.Diagnostics;
 using Elastic.Documentation.Search;
 using Elastic.Documentation.Serialization;
@@ -20,23 +19,24 @@
 
 namespace Elastic.Markdown.Exporters;
 
-public class ElasticsearchMarkdownExporter(ILoggerFactory logFactory, IDiagnosticsCollector collector, DocumentationEndpoints endpoints)
+public class ElasticsearchMarkdownExporter(ILoggerFactory logFactory, IDiagnosticsCollector collector, string indexNamespace, DocumentationEndpoints endpoints)
 	: ElasticsearchMarkdownExporterBase<CatalogIndexChannelOptions<DocumentationDocument>, CatalogIndexChannel<DocumentationDocument>>
 		(logFactory, collector, endpoints)
 {
 	/// <inheritdoc />
 	protected override CatalogIndexChannelOptions<DocumentationDocument> NewOptions(DistributedTransport transport) => new(transport)
 	{
 		GetMapping = () => CreateMapping(null),
-		IndexFormat = "documentation{0:yyyy.MM.dd.HHmmss}",
-		ActiveSearchAlias = "documentation"
+		GetMappingSettings = () => CreateMappingSetting(),
+		IndexFormat = $"{Endpoint.IndexNamePrefix.ToLowerInvariant()}-{indexNamespace.ToLowerInvariant()}-{{0:yyyy.MM.dd.HHmmss}}",
+		ActiveSearchAlias = $"{Endpoint.IndexNamePrefix}-{indexNamespace.ToLowerInvariant()}",
 	};
 
 	/// <inheritdoc />
 	protected override CatalogIndexChannel<DocumentationDocument> NewChannel(CatalogIndexChannelOptions<DocumentationDocument> options) => new(options);
 }
 
-public class ElasticsearchMarkdownSemanticExporter(PublishEnvironment environment, ILoggerFactory logFactory, IDiagnosticsCollector collector, DocumentationEndpoints endpoints)
+public class ElasticsearchMarkdownSemanticExporter(ILoggerFactory logFactory, IDiagnosticsCollector collector, string indexNamespace, DocumentationEndpoints endpoints)
 	: ElasticsearchMarkdownExporterBase<SemanticIndexChannelOptions<DocumentationDocument>, SemanticIndexChannel<DocumentationDocument>>
 		(logFactory, collector, endpoints)
 {
@@ -45,20 +45,23 @@ public class ElasticsearchMarkdownSemanticExporter(PublishEnvironment environmen
 	{
 		GetMapping = (inferenceId, _) => CreateMapping(inferenceId),
 		GetMappingSettings = (_, _) => CreateMappingSetting(),
-		IndexFormat = $"semantic-docs-{environment.Name}-{{0:yyyy.MM.dd.HHmmss}}",
-		ActiveSearchAlias = $"semantic-docs-{environment.Name}",
-		IndexNumThreads = IndexNumThreads,
-		InferenceCreateTimeout = TimeSpan.FromMinutes(4)
+		IndexFormat = $"{Endpoint.IndexNamePrefix.ToLowerInvariant()}-{indexNamespace.ToLowerInvariant()}-{{0:yyyy.MM.dd.HHmmss}}",
+		ActiveSearchAlias = $"{Endpoint.IndexNamePrefix}-{indexNamespace.ToLowerInvariant()}",
+		IndexNumThreads = Endpoint.IndexNumThreads,
+		SearchNumThreads = Endpoint.SearchNumThreads,
+		InferenceCreateTimeout = TimeSpan.FromMinutes(Endpoint.BootstrapTimeout ?? 4)
 	};
 
 	/// <inheritdoc />
 	protected override SemanticIndexChannel<DocumentationDocument> NewChannel(SemanticIndexChannelOptions<DocumentationDocument> options) => new(options);
 }
 
+
 public abstract class ElasticsearchMarkdownExporterBase<TChannelOptions, TChannel>(
 	ILoggerFactory logFactory,
 	IDiagnosticsCollector collector,
-	DocumentationEndpoints endpoints)
+	DocumentationEndpoints endpoints
+)
 	: IMarkdownExporter, IDisposable
 	where TChannelOptions : CatalogIndexChannelOptionsBase<DocumentationDocument>
 	where TChannel : CatalogIndexChannel<DocumentationDocument, TChannelOptions>
@@ -69,7 +72,7 @@ public abstract class ElasticsearchMarkdownExporterBase<TChannelOptions, TChanne
 	protected abstract TChannelOptions NewOptions(DistributedTransport transport);
 	protected abstract TChannel NewChannel(TChannelOptions options);
 
-	protected int IndexNumThreads => 8;
+	protected ElasticsearchEndpoint Endpoint { get; } = endpoints.Elasticsearch;
 
 	protected static string CreateMappingSetting() =>
 		// language=json
@@ -97,7 +100,6 @@ protected static string CreateMappingSetting() =>
 		""";
 
 	protected static string CreateMapping(string? inferenceId) =>
-		// langugage=json
 		$$"""
 		  {
 		    "properties": {
@@ -131,15 +133,13 @@ protected static string CreateMapping(string? inferenceId) =>
 		  """;
 
 	private static string AbstractMapping() =>
-		// langugage=json
 		"""
 		, "abstract": {
 			"type": "text"
 		}
 		""";
 
 	private static string InferenceMapping(string inferenceId) =>
-		// langugage=json
 		$"""
 		 	"type": "semantic_text",
 		 	"inference_id": "{inferenceId}"
@@ -159,12 +159,26 @@ public async ValueTask StartAsync(Cancel ctx = default)
 			return;
 
 		var es = endpoints.Elasticsearch;
+
 		var configuration = new ElasticsearchConfiguration(es.Uri)
 		{
 			Authentication = es.ApiKey is { } apiKey
 				? new ApiKey(apiKey)
-				: es.Username is { } username && es.Password is { } password
+				: es is { Username: { } username, Password: { } password }
 					? new BasicAuthentication(username, password)
+					: null,
+			EnableHttpCompression = true,
+			DebugMode = Endpoint.DebugMode,
+			CertificateFingerprint = Endpoint.CertificateFingerprint,
+			ProxyAddress = Endpoint.ProxyAddress,
+			ProxyPassword = Endpoint.ProxyPassword,
+			ProxyUsername = Endpoint.ProxyUsername,
+			ServerCertificateValidationCallback = Endpoint.DisableSslVerification
+				? CertificateValidations.AllowAll
+				: Endpoint.Certificate is { } cert
+					? Endpoint.CertificateIsNotRoot
+						? CertificateValidations.AuthorityPartOfChain(cert)
+						: CertificateValidations.AuthorityIsRoot(cert)
 					: null
 		};
 
@@ -173,14 +187,20 @@ public async ValueTask StartAsync(Cancel ctx = default)
 		//The max num threads per allocated node, from testing its best to limit our max concurrency
 		//producing to this number as well
 		var options = NewOptions(transport);
+		var i = 0;
 		options.BufferOptions = new BufferOptions
 		{
-			OutboundBufferMaxSize = 100,
-			ExportMaxConcurrency = IndexNumThreads,
-			ExportMaxRetries = 3
+			OutboundBufferMaxSize = Endpoint.BufferSize,
+			ExportMaxConcurrency = Endpoint.IndexNumThreads,
+			ExportMaxRetries = Endpoint.MaxRetries,
 		};
 		options.SerializerContext = SourceGenerationContext.Default;
-		options.ExportBufferCallback = () => _logger.LogInformation("Exported buffer to Elasticsearch");
+		options.ExportBufferCallback = () =>
+		{
+			var count = Interlocked.Increment(ref i);
+			_logger.LogInformation("Exported {Count} documents to Elasticsearch index {Format}",
+				count * Endpoint.BufferSize, options.IndexFormat);
+		};
 		options.ExportExceptionCallback = e => _logger.LogError(e, "Failed to export document");
 		options.ServerRejectionCallback = items => _logger.LogInformation("Server rejection: {Rejection}", items.First().Item2);
 		_channel = NewChannel(options);
@@ -206,7 +226,7 @@ public async ValueTask StopAsync(Cancel ctx = default)
 		_logger.LogInformation("Applying aliases to {Index}", _channel.IndexName);
 		var swapped = await _channel.ApplyAliasesAsync(ctx);
 		if (!swapped)
-			collector.EmitGlobalError($"{nameof(ElasticsearchMarkdownExporter)} failed to apply aliases to index {_channel.IndexName}");
+			collector.EmitGlobalError($"${nameof(ElasticsearchMarkdownExporter)} failed to apply aliases to index {_channel.IndexName}");
 	}
 
 	public void Dispose()

src/Elastic.Markdown/Exporters/ExporterExtensions.cs

@@ -15,7 +15,7 @@ public static IReadOnlyCollection<IMarkdownExporter> CreateMarkdownExporters(
 		this IReadOnlySet<Exporter> exportOptions,
 		ILoggerFactory logFactory,
 		IDocumentationConfigurationContext context,
-		PublishEnvironment? environment = null
+		string indexNamespace
 	)
 	{
 		var markdownExporters = new List<IMarkdownExporter>(3);
@@ -24,13 +24,9 @@ public static IReadOnlyCollection<IMarkdownExporter> CreateMarkdownExporters(
 		if (exportOptions.Contains(Exporter.Configuration))
 			markdownExporters.Add(new ConfigurationExporter(logFactory, context.ConfigurationFileProvider, context));
 		if (exportOptions.Contains(Exporter.Elasticsearch))
-			markdownExporters.Add(new ElasticsearchMarkdownExporter(logFactory, context.Collector, context.Endpoints));
-		if (exportOptions.Contains(Exporter.SemanticElasticsearch))
-		{
-			if (environment is null)
-				throw new ArgumentNullException(nameof(environment), "A publish environment is required when using the semantic elasticsearch exporter");
-			markdownExporters.Add(new ElasticsearchMarkdownSemanticExporter(environment, logFactory, context.Collector, context.Endpoints));
-		}
+			markdownExporters.Add(new ElasticsearchMarkdownSemanticExporter(logFactory, context.Collector, indexNamespace, context.Endpoints));
+		if (exportOptions.Contains(Exporter.ElasticsearchNoSemantic))
+			markdownExporters.Add(new ElasticsearchMarkdownExporter(logFactory, context.Collector, indexNamespace, context.Endpoints));
 		return markdownExporters;
 	}
 }

src/authoring/Elastic.Documentation.Refactor/Tracking/LocalChangesService.cs

@@ -19,7 +19,7 @@ IConfigurationContext configurationContext
 {
 	private readonly ILogger _logger = logFactory.CreateLogger<LocalChangeTrackingService>();
 
-	public async Task<bool> ValidateRedirects(IDiagnosticsCollector collector, string? path, FileSystem fs, Cancel ctx)
+	public Task<bool> ValidateRedirects(IDiagnosticsCollector collector, string? path, FileSystem fs)
 	{
 		var runningOnCi = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("GITHUB_ACTIONS"));
 
@@ -28,21 +28,21 @@ public async Task<bool> ValidateRedirects(IDiagnosticsCollector collector, strin
 		if (!redirectFile.Source.Exists)
 		{
 			collector.EmitError(redirectFile.Source, "File does not exist");
-			return false;
+			return Task.FromResult(false);
 		}
 
 		var redirects = redirectFile.Redirects;
 		if (redirects is null)
 		{
 			collector.EmitError(redirectFile.Source, "It was not possible to parse the redirects file.");
-			return false;
+			return Task.FromResult(false);
 		}
 
 		var root = Paths.DetermineSourceDirectoryRoot(buildContext.DocumentationSourceDirectory);
 		if (root is null)
 		{
 			collector.EmitError(redirectFile.Source, $"Unable to determine the root of the source directory {buildContext.DocumentationSourceDirectory}.");
-			return false;
+			return Task.FromResult(false);
 		}
 		var relativePath = Path.GetRelativePath(root.FullName, buildContext.DocumentationSourceDirectory.FullName);
 		_logger.LogInformation("Using relative path {RelativePath} for validating changes", relativePath);
@@ -87,7 +87,6 @@ public async Task<bool> ValidateRedirects(IDiagnosticsCollector collector, strin
 			_logger.LogInformation("Found {Count} changes that still require updates to: {RedirectFile}", missingCount, relativeRedirectFile);
 		}
 
-		await collector.StopAsync(ctx);
-		return collector.Errors == 0;
+		return Task.FromResult(collector.Errors == 0);
 	}
 }