Add support for diagnostic hints and track unused substitution keys (#762)

* Add support for diagnostic hints and track unused substitution keys

Introduced a "Hint" severity level to diagnostics for improvement suggestions. Enhanced substitution tracking to collect unused keys and provide hints, optimizing substitution usage. Updated console and GitHub outputs to display diagnostic hints alongside errors and warnings for better feedback.

* update hint errate display

Author: Mpdreamz

src/Elastic.Documentation.Tooling/Diagnostics/Console/ConsoleDiagnosticsCollector.cs

@@ -16,13 +16,16 @@ public class ConsoleDiagnosticsCollector(ILoggerFactory loggerFactory, ICoreServ
 {
 	private readonly List<Diagnostic> _errors = [];
 	private readonly List<Diagnostic> _warnings = [];
+	private readonly List<Diagnostic> _hints = [];
 
 	protected override void HandleItem(Diagnostic diagnostic)
 	{
-		if (diagnostic.Severity == Severity.Warning)
+		if (diagnostic.Severity == Severity.Error)
+			_errors.Add(diagnostic);
+		else if (diagnostic.Severity == Severity.Warning)
 			_warnings.Add(diagnostic);
 		else
-			_errors.Add(diagnostic);
+			_hints.Add(diagnostic);
 	}
 
 	private bool _stopped;
@@ -32,10 +35,10 @@ public override async Task StopAsync(Cancel cancellationToken)
 			return;
 		_stopped = true;
 		var repository = new ErrataFileSourceRepository();
-		repository.WriteDiagnosticsToConsole(_errors, _warnings);
+		repository.WriteDiagnosticsToConsole(_errors, _warnings, _hints);
 
 		AnsiConsole.WriteLine();
-		AnsiConsole.Write(new Markup($"	[bold red]{Errors} Errors[/] / [bold blue]{Warnings} Warnings[/]"));
+		AnsiConsole.Write(new Markup($"	[bold red]{Errors} Errors[/] / [bold blue]{Warnings} Warnings[/] / [bold yellow]{Hints} Hints[/]"));
 		AnsiConsole.WriteLine();
 		AnsiConsole.WriteLine();
 

src/Elastic.Documentation.Tooling/Diagnostics/Console/ErrataFileSourceRepository.cs

@@ -23,19 +23,26 @@ public bool TryGet(string id, [NotNullWhen(true)] out Source? source)
 		return true;
 	}
 
-	public void WriteDiagnosticsToConsole(IReadOnlyCollection<Diagnostic> errors, IReadOnlyCollection<Diagnostic> warnings)
+	public void WriteDiagnosticsToConsole(IReadOnlyCollection<Diagnostic> errors, IReadOnlyCollection<Diagnostic> warnings, List<Diagnostic> hints)
 	{
 		var report = new Report(this);
-		var limitedErrors = errors.Take(100).ToArray();
-		var limitedWarnings = warnings.Take(100 - limitedErrors.Length);
-		var limited = limitedWarnings.Concat(limitedErrors).ToArray();
+		var limited = errors
+			.Concat(warnings)
+			.OrderBy(d => d.Severity switch { Severity.Error => 0, Severity.Warning => 1, Severity.Hint => 2, _ => 3 })
+			.Take(100)
+			.ToArray();
+
+		// show hints if we don't have plenty of errors/warnings to show
+		if (limited.Length < 100)
+			limited = limited.Concat(hints).Take(100).ToArray();
 
 		foreach (var item in limited)
 		{
 			var d = item.Severity switch
 			{
 				Severity.Error => Errata.Diagnostic.Error(item.Message),
 				Severity.Warning => Errata.Diagnostic.Warning(item.Message),
+				Severity.Hint => Errata.Diagnostic.Info(item.Message),
 				_ => Errata.Diagnostic.Info(item.Message)
 			};
 			if (item is { Line: not null, Column: not null })
@@ -44,19 +51,54 @@ public void WriteDiagnosticsToConsole(IReadOnlyCollection<Diagnostic> errors, IR
 				d = d.WithLabel(new Label(item.File, location, "")
 					.WithLength(item.Length == null ? 1 : Math.Clamp(item.Length.Value, 1, item.Length.Value + 3))
 					.WithPriority(1)
-					.WithColor(item.Severity == Severity.Error ? Color.Red : Color.Blue));
+					.WithColor(item.Severity switch
+					{
+						Severity.Error => Color.Red,
+						Severity.Warning => Color.Blue,
+						Severity.Hint => Color.Yellow,
+						_ => Color.Blue
+					}));
 			}
 			else
 				d = d.WithNote(item.File);
 
+			if (item.Severity == Severity.Hint)
+				d = d.WithColor(Color.Yellow).WithCategory("Hint");
+
 			_ = report.AddDiagnostic(d);
 		}
 
 		var totalErrorCount = errors.Count + warnings.Count;
 
 		AnsiConsole.WriteLine();
 		if (totalErrorCount <= 0)
+		{
+			if (hints.Count > 0)
+				DisplayHintsOnly(report, hints);
 			return;
+		}
+		DisplayErrorAndWarningSummary(report, totalErrorCount, limited);
+	}
+
+	private static void DisplayHintsOnly(Report report, List<Diagnostic> hints)
+	{
+		AnsiConsole.Write(new Markup($"	[bold]The following improvement hints found in the documentation[/]"));
+		AnsiConsole.WriteLine();
+		AnsiConsole.WriteLine();
+		// Render the report
+		report.Render(AnsiConsole.Console);
+
+		AnsiConsole.WriteLine();
+		AnsiConsole.WriteLine();
+
+		if (hints.Count >= 100)
+			AnsiConsole.Write(new Markup($"	[bold]Only shown the first [yellow]{100}[/] hints out of [yellow]{hints.Count}[/][/]"));
+
+		AnsiConsole.WriteLine();
+	}
+
+	private static void DisplayErrorAndWarningSummary(Report report, int totalErrorCount, Diagnostic[] limited)
+	{
 		AnsiConsole.Write(new Markup($"	[bold]The following errors and warnings were found in the documentation[/]"));
 		AnsiConsole.WriteLine();
 		AnsiConsole.WriteLine();

src/Elastic.Documentation.Tooling/Diagnostics/Console/GithubAnnotationOutput.cs

@@ -27,5 +27,7 @@ public void Write(Diagnostic diagnostic)
 			githubActions.WriteError(diagnostic.Message, properties);
 		if (diagnostic.Severity == Severity.Warning)
 			githubActions.WriteWarning(diagnostic.Message, properties);
+		if (diagnostic.Severity == Severity.Hint)
+			githubActions.WriteNotice(diagnostic.Message, properties);
 	}
 }

src/Elastic.Documentation.Tooling/Diagnostics/Log.cs

@@ -16,15 +16,19 @@ public void Write(Diagnostic diagnostic)
 		{
 			if (diagnostic.Severity == Severity.Error)
 				logger.LogError("{Message}", diagnostic.Message);
-			else
+			else if (diagnostic.Severity == Severity.Warning)
 				logger.LogWarning("{Message}", diagnostic.Message);
+			else
+				logger.LogInformation("{Message}", diagnostic.Message);
 		}
 		else
 		{
 			if (diagnostic.Severity == Severity.Error)
 				logger.LogError("{Message} ({File}:{Line})", diagnostic.Message, diagnostic.File, diagnostic.Line ?? 0);
+			else if (diagnostic.Severity == Severity.Warning)
+				logger.LogWarning("{Message}", diagnostic.Message);
 			else
-				logger.LogWarning("{Message} ({File}:{Line})", diagnostic.Message, diagnostic.File, diagnostic.Line ?? 0);
+				logger.LogInformation("{Message} ({File}:{Line})", diagnostic.Message, diagnostic.File, diagnostic.Line ?? 0);
 		}
 	}
 }

src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs

@@ -8,6 +8,23 @@
 
 namespace Elastic.Markdown.Diagnostics;
 
+public enum Severity
+{
+	Error,
+	Warning,
+	Hint
+}
+
+public readonly record struct Diagnostic
+{
+	public Severity Severity { get; init; }
+	public int? Line { get; init; }
+	public int? Column { get; init; }
+	public int? Length { get; init; }
+	public string File { get; init; }
+	public string Message { get; init; }
+}
+
 public sealed class DiagnosticsChannel : IDisposable
 {
 	private readonly Channel<Diagnostic> _channel;
@@ -18,7 +35,11 @@ public sealed class DiagnosticsChannel : IDisposable
 
 	public DiagnosticsChannel()
 	{
-		var options = new UnboundedChannelOptions { SingleReader = true, SingleWriter = false };
+		var options = new UnboundedChannelOptions
+		{
+			SingleReader = true,
+			SingleWriter = false
+		};
 		_ctxSource = new CancellationTokenSource();
 		_channel = Channel.CreateUnbounded<Diagnostic>(options);
 	}
@@ -43,18 +64,6 @@ public void Write(Diagnostic diagnostic)
 	public void Dispose() => _ctxSource.Dispose();
 }
 
-public enum Severity { Error, Warning }
-
-public readonly record struct Diagnostic
-{
-	public Severity Severity { get; init; }
-	public int? Line { get; init; }
-	public int? Column { get; init; }
-	public int? Length { get; init; }
-	public string File { get; init; }
-	public string Message { get; init; }
-}
-
 public interface IDiagnosticsOutput
 {
 	void Write(Diagnostic diagnostic);
@@ -67,13 +76,17 @@ public class DiagnosticsCollector(IReadOnlyCollection<IDiagnosticsOutput> output
 
 	private int _errors;
 	private int _warnings;
+	private int _hints;
 	public int Warnings => _warnings;
 	public int Errors => _errors;
+	public int Hints => _hints;
 
 	private Task? _started;
 
 	public HashSet<string> OffendingFiles { get; } = [];
 
+	public HashSet<string> InUseSubstitutionKeys { get; } = [];
+
 	public ConcurrentBag<string> CrossLinks { get; } = [];
 
 	public Task StartAsync(Cancel cancellationToken)
@@ -119,6 +132,8 @@ private void IncrementSeverityCount(Diagnostic item)
 			_ = Interlocked.Increment(ref _errors);
 		else if (item.Severity == Severity.Warning)
 			_ = Interlocked.Increment(ref _warnings);
+		else if (item.Severity == Severity.Hint)
+			_ = Interlocked.Increment(ref _hints);
 	}
 
 	protected virtual void HandleItem(Diagnostic diagnostic) { }
@@ -132,35 +147,33 @@ public virtual async Task StopAsync(Cancel cancellationToken)
 
 	public void EmitCrossLink(string link) => CrossLinks.Add(link);
 
-	public void EmitError(string file, string message, Exception? e = null)
-	{
-		var d = new Diagnostic
+	private void Emit(Severity severity, string file, string message) =>
+		Channel.Write(new Diagnostic
 		{
-			Severity = Severity.Error,
+			Severity = severity,
 			File = file,
 			Message = message
-					  + (e != null ? Environment.NewLine + e : string.Empty)
-					  + (e?.InnerException != null ? Environment.NewLine + e.InnerException : string.Empty),
+		});
 
-		};
-		Channel.Write(d);
-	}
-
-	public void EmitWarning(string file, string message)
+	public void EmitError(string file, string message, Exception? e = null)
 	{
-		var d = new Diagnostic
-		{
-			Severity = Severity.Warning,
-			File = file,
-			Message = message,
-		};
-		Channel.Write(d);
+		message = message
+				+ (e != null ? Environment.NewLine + e : string.Empty)
+				+ (e?.InnerException != null ? Environment.NewLine + e.InnerException : string.Empty);
+		Emit(Severity.Error, file, message);
 	}
 
+	public void EmitWarning(string file, string message) => Emit(Severity.Warning, file, message);
+
+	public void EmitHint(string file, string message) => Emit(Severity.Hint, file, message);
+
 	public async ValueTask DisposeAsync()
 	{
 		Channel.TryComplete();
 		await StopAsync(CancellationToken.None);
 		GC.SuppressFinalize(this);
 	}
+
+	public void CollectUsedSubstitutionKey(ReadOnlySpan<char> key) =>
+		_ = InUseSubstitutionKeys.Add(key.ToString());
 }

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -87,6 +87,8 @@ public async Task GenerateAll(Cancel ctx)
 
 		await ProcessDocumentationFiles(offendingFiles, outputSeenChanges, ctx);
 
+		HintUnusedSubstitutionKeys();
+
 		await ExtractEmbeddedStaticResources(ctx);
 
 		_logger.LogInformation($"Completing diagnostics channel");
@@ -135,6 +137,24 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) =>
 		_logger.LogInformation("-> Processed {ProcessedFileCount}/{TotalFileCount} files", processedFileCount, totalFileCount);
 	}
 
+	private void HintUnusedSubstitutionKeys()
+	{
+		var definedKeys = new HashSet<string>(Context.Configuration.Substitutions.Keys.ToArray());
+		var keysNotInUse = definedKeys.Except(Context.Collector.InUseSubstitutionKeys).ToArray();
+		// If we have less than 20 unused keys emit them separately
+		// Otherwise emit one hint with all of them for brevity
+		if (keysNotInUse.Length >= 20)
+		{
+			var keys = string.Join(", ", keysNotInUse);
+			Context.Collector.EmitHint(Context.ConfigurationPath.FullName, $"The following keys: '{keys}' are not used in any file");
+		}
+		else
+		{
+			foreach (var key in keysNotInUse)
+				Context.Collector.EmitHint(Context.ConfigurationPath.FullName, $"Substitution key '{key}' is not used in any file");
+		}
+	}
+
 	private async Task ExtractEmbeddedStaticResources(Cancel ctx)
 	{
 		_logger.LogInformation($"Copying static files to output directory");

src/Elastic.Markdown/Helpers/Interpolation.cs

@@ -4,6 +4,7 @@
 
 using System.Diagnostics.CodeAnalysis;
 using System.Text.RegularExpressions;
+using Elastic.Markdown.Diagnostics;
 using Elastic.Markdown.Myst;
 
 namespace Elastic.Markdown.Helpers;
@@ -22,32 +23,26 @@ ParserContext context
 	)
 	{
 		var span = input.AsSpan();
-		return span.ReplaceSubstitutions([context.Substitutions, context.ContextSubstitutions], out var replacement)
+		return span.ReplaceSubstitutions([context.Substitutions, context.ContextSubstitutions], context.Build.Collector, out var replacement)
 			? replacement : input;
 	}
 
-
-	public static bool ReplaceSubstitutions(
-		this ReadOnlySpan<char> span,
-		ParserContext context,
-		[NotNullWhen(true)] out string? replacement
-	) =>
-		span.ReplaceSubstitutions([context.Substitutions, context.ContextSubstitutions], out replacement);
-
 	public static bool ReplaceSubstitutions(
 		this ReadOnlySpan<char> span,
 		IReadOnlyDictionary<string, string>? properties,
+		DiagnosticsCollector? collector,
 		[NotNullWhen(true)] out string? replacement
 	)
 	{
 		replacement = null;
 		return properties is not null && properties.Count != 0 &&
-			span.IndexOf("}}") >= 0 && span.ReplaceSubstitutions([properties], out replacement);
+			span.IndexOf("}}") >= 0 && span.ReplaceSubstitutions([properties], collector, out replacement);
 	}
 
-	public static bool ReplaceSubstitutions(
+	private static bool ReplaceSubstitutions(
 		this ReadOnlySpan<char> span,
 		IReadOnlyDictionary<string, string>[] properties,
+		DiagnosticsCollector? collector,
 		[NotNullWhen(true)] out string? replacement
 	)
 	{
@@ -78,6 +73,8 @@ public static bool ReplaceSubstitutions(
 				if (!lookup.TryGetValue(key, out var value))
 					continue;
 
+				collector?.CollectUsedSubstitutionKey(key);
+
 				replacement ??= span.ToString();
 				replacement = replacement.Replace(spanMatch.ToString(), value);
 				replaced = true;

src/Elastic.Markdown/IO/MarkdownFile.cs

@@ -210,7 +210,7 @@ protected void ReadDocumentInstructions(MarkdownDocument document)
 
 		if (!string.IsNullOrEmpty(NavigationTitle))
 		{
-			if (NavigationTitle.AsSpan().ReplaceSubstitutions(subs, out var replacement))
+			if (NavigationTitle.AsSpan().ReplaceSubstitutions(subs, Collector, out var replacement))
 				NavigationTitle = replacement;
 		}
 
@@ -219,7 +219,7 @@ protected void ReadDocumentInstructions(MarkdownDocument document)
 			Title = RelativePath;
 			Collector.EmitWarning(FilePath, "Document has no title, using file name as title.");
 		}
-		else if (Title.AsSpan().ReplaceSubstitutions(subs, out var replacement))
+		else if (Title.AsSpan().ReplaceSubstitutions(subs, Collector, out var replacement))
 			Title = replacement;
 
 		var toc = GetAnchors(_set, MarkdownParser, YamlFrontMatter, document, subs, out var anchors);
@@ -272,7 +272,7 @@ public static List<PageTocItem> GetAnchors(
 			.Concat(includedTocs)
 			.Select(toc => subs.Count == 0
 				? toc
-				: toc.Heading.AsSpan().ReplaceSubstitutions(subs, out var r)
+				: toc.Heading.AsSpan().ReplaceSubstitutions(subs, set.Build.Collector, out var r)
 					? toc with { Heading = r }
 					: toc)
 			.ToList();