Limit diagnostics output and enhance logging clarity (#208)

Author: Mpdreamz

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -88,10 +88,11 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) =>
 					throw;
 			}
 
-			if (processedFiles % 1_000 == 0)
-				_logger.LogInformation($"Handled {processedFiles} files");
+			if (processedFiles % 100 == 0)
+				_logger.LogInformation($"-> Handled {processedFiles} files");
 		});
 
+		_logger.LogInformation($"Copying static files to output directory");
 		var embeddedStaticFiles = Assembly.GetExecutingAssembly()
 			.GetManifestResourceNames()
 			.ToList();
@@ -112,12 +113,19 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) =>
 		}
 
 
+		_logger.LogInformation($"Completing diagnostics channel");
 		Context.Collector.Channel.TryComplete();
 
+		_logger.LogInformation($"Generating documentation compilation state");
 		await GenerateDocumentationState(ctx);
+		_logger.LogInformation($"Generating links.json");
 		await GenerateLinkReference(ctx);
 
+		_logger.LogInformation($"Completing diagnostics channel");
+
 		await Context.Collector.StopAsync(ctx);
+
+		_logger.LogInformation($"Completed diagnostics channel");
 	}
 
 	private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile file, DateTimeOffset outputSeenChanges, CancellationToken token)

src/Elastic.Markdown/Helpers/Interpolation.cs

@@ -17,6 +17,9 @@ public static class Interpolation
 	public static bool ReplaceSubstitutions(this ReadOnlySpan<char> span, Dictionary<string, string>? properties, out string? replacement)
 	{
 		replacement = null;
+		if (span.IndexOf("}}") < 0)
+			return false;
+
 		var substitutions = properties ?? new();
 		if (substitutions.Count == 0)
 			return false;

src/Elastic.Markdown/IO/Discovery/GitCheckoutInformation.cs

@@ -40,9 +40,9 @@ public string? RepositoryName
 	public static GitCheckoutInformation Create(IFileSystem fileSystem)
 	{
 		// filesystem is not real so return a dummy
+		var fakeRef = Guid.NewGuid().ToString().Substring(0, 16);
 		if (fileSystem is not FileSystem)
 		{
-			var fakeRef = Guid.NewGuid().ToString().Substring(0, 16);
 			return new GitCheckoutInformation
 			{
 				Branch = $"test-{fakeRef}",
@@ -56,14 +56,14 @@ public static GitCheckoutInformation Create(IFileSystem fileSystem)
 		if (!gitConfig.Exists)
 			return Unavailable;
 
-		var head = Read(".git/HEAD");
+		var head = Read(".git/HEAD") ?? fakeRef;
 		var gitRef = head;
 		var branch = head.Replace("refs/heads/", string.Empty);
 		//not detached HEAD
 		if (head.StartsWith("ref:"))
 		{
 			head = head.Replace("ref: ", string.Empty);
-			gitRef = Read(".git/" + head);
+			gitRef = Read(".git/" + head) ?? fakeRef;
 			branch = branch.Replace("ref: ", string.Empty);
 		}
 		else
@@ -94,8 +94,13 @@ public static GitCheckoutInformation Create(IFileSystem fileSystem)
 
 		IFileInfo Git(string path) => fileSystem.FileInfo.New(Path.Combine(Paths.Root.FullName, path));
 
-		string Read(string path) =>
-			fileSystem.File.ReadAllText(Git(path).FullName).Trim(Environment.NewLine.ToCharArray());
+		string? Read(string path)
+		{
+			var gitPath = Git(path).FullName;
+			if (!fileSystem.File.Exists(gitPath))
+				return null;
+			return fileSystem.File.ReadAllText(gitPath).Trim(Environment.NewLine.ToCharArray());
+		}
 
 		string BranchTrackingRemote(string b, IniFile c)
 		{

src/Elastic.Markdown/Myst/CodeBlocks/CallOutParser.cs

@@ -11,6 +11,6 @@ public static partial class CallOutParser
 	[GeneratedRegex(@"^.+\S+.*?\s<\d+>$", RegexOptions.IgnoreCase, "en-US")]
 	public static partial Regex CallOutNumber();
 
-	[GeneratedRegex(@"^.+\S+.*?\s(?:\/\/|#)\s[^""]+$", RegexOptions.IgnoreCase, "en-US")]
+	[GeneratedRegex(@"^.+\S+.*?\s(?:\/\/|#)\s[^""\/#]+$", RegexOptions.IgnoreCase, "en-US")]
 	public static partial Regex MathInlineAnnotation();
 }

src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockParser.cs

@@ -98,10 +98,19 @@ public override bool Close(BlockProcessor processor, Block block)
 			if (codeBlock.OpeningFencedCharCount > 3)
 				continue;
 
-			var matchClassicCallout = CallOutParser.CallOutNumber().EnumerateMatches(span);
-			var callOut = EnumerateAnnotations(matchClassicCallout, ref span, ref callOutIndex, originatingLine, false);
+			if (span.IndexOf("<") < 0 && span.IndexOf("//") < 0)
+				continue;
 
-			if (callOut is null)
+			CallOut? callOut = null;
+
+			if (span.IndexOf("<") > 0)
+			{
+				var matchClassicCallout = CallOutParser.CallOutNumber().EnumerateMatches(span);
+				callOut = EnumerateAnnotations(matchClassicCallout, ref span, ref callOutIndex, originatingLine, false);
+			}
+
+			// only support magic callouts for smaller line lengths
+			if (callOut is null && span.Length < 200)
 			{
 				var matchInline = CallOutParser.MathInlineAnnotation().EnumerateMatches(span);
 				callOut = EnumerateAnnotations(matchInline, ref span, ref callOutIndex, originatingLine,

src/docs-builder/Diagnostics/Console/ConsoleDiagnosticsCollector.cs

@@ -14,14 +14,21 @@ public class ConsoleDiagnosticsCollector(ILoggerFactory loggerFactory, ICoreServ
 	: DiagnosticsCollector([new Log(loggerFactory.CreateLogger<Log>()), new GithubAnnotationOutput(githubActions)]
 	)
 {
-	private readonly List<Diagnostic> _items = new();
+	private readonly List<Diagnostic> _errors = new();
+	private readonly List<Diagnostic> _warnings = new();
 
-	protected override void HandleItem(Diagnostic diagnostic) => _items.Add(diagnostic);
+	protected override void HandleItem(Diagnostic diagnostic)
+	{
+		if (diagnostic.Severity == Severity.Warning)
+			_warnings.Add(diagnostic);
+		else
+			_errors.Add(diagnostic);
+	}
 
 	public override async Task StopAsync(Cancel ctx)
 	{
 		var repository = new ErrataFileSourceRepository();
-		repository.WriteDiagnosticsToConsole(_items);
+		repository.WriteDiagnosticsToConsole(_errors, _warnings);
 
 		AnsiConsole.WriteLine();
 		AnsiConsole.Write(new Markup($"	[bold red]{Errors} Errors[/] / [bold blue]{Warnings} Warnings[/]"));

src/docs-builder/Diagnostics/Console/ErrataFileSourceRepository.cs

@@ -22,10 +22,14 @@ public bool TryGet(string id, [NotNullWhen(true)] out Source? source)
 		return true;
 	}
 
-	public void WriteDiagnosticsToConsole(IReadOnlyCollection<Diagnostic> items)
+	public void WriteDiagnosticsToConsole(IReadOnlyCollection<Diagnostic> errors, IReadOnlyCollection<Diagnostic> warnings)
 	{
 		var report = new Report(this);
-		foreach (var item in items)
+		var limttedErrors = errors.Take(100).ToArray();
+		var limittedWarnings = warnings.Take(100 - limttedErrors.Length);
+		var limitted = limittedWarnings.Concat(limttedErrors).ToArray();
+
+		foreach (var item in limitted)
 		{
 			var d = item.Severity switch
 			{
@@ -49,5 +53,15 @@ public void WriteDiagnosticsToConsole(IReadOnlyCollection<Diagnostic> items)
 
 		// Render the report
 		report.Render(AnsiConsole.Console);
+
+		var totalErrorCount = errors.Count + warnings.Count;
+		if (limitted.Length >= totalErrorCount)
+			return;
+
+		AnsiConsole.WriteLine();
+		AnsiConsole.WriteLine();
+		AnsiConsole.Write(new Markup($"Displayed first [bold]{limitted.Length}[/] error/warnings out of [bold]{totalErrorCount}[/]"));
+
+		AnsiConsole.WriteLine();
 	}
 }