From 8ac3e10e5e62477dcc4a59e5e1d6effefe39edf0 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 15 Jan 2025 12:44:59 +0100 Subject: [PATCH 1/3] Limit diagnostics output and enhance logging clarity Introduced pagination for diagnostics console output to handle only the first 100 errors/warnings and added clear messages for truncated results. Improved logging consistency in various components and added null safety when reading Git-related data. This PR also caught a performance regression in the handling of codeblocks. It was too eagerly trying to attempt substitutions on each line. --- .../Diagnostics/DiagnosticsChannel.cs | 1 + src/Elastic.Markdown/DocumentationGenerator.cs | 12 ++++++++++-- .../IO/Discovery/GitCheckoutInformation.cs | 15 ++++++++++----- .../Myst/CodeBlocks/EnhancedCodeBlockParser.cs | 2 ++ .../Console/ConsoleDiagnosticsCollector.cs | 13 ++++++++++--- .../Console/ErrataFileSourceRepository.cs | 18 ++++++++++++++++-- 6 files changed, 49 insertions(+), 12 deletions(-) diff --git a/src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs b/src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs index d6848520d..1f1389533 100644 --- a/src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs +++ b/src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs @@ -124,6 +124,7 @@ protected virtual void HandleItem(Diagnostic diagnostic) { } public virtual async Task StopAsync(CancellationToken cancellationToken) { + Console.WriteLine("Stopping diagnostics"); if (_started is not null) await _started; await Channel.Reader.Completion; diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs index cf8208290..bb6f71a24 100644 --- a/src/Elastic.Markdown/DocumentationGenerator.cs +++ b/src/Elastic.Markdown/DocumentationGenerator.cs @@ -88,10 +88,11 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) => throw; } - if (processedFiles % 1_000 == 0) - _logger.LogInformation($"Handled {processedFiles} files"); + if (processedFiles % 100 == 0) + _logger.LogInformation($"-> Handled {processedFiles} files"); }); + _logger.LogInformation($"Copying static files to output directory"); var embeddedStaticFiles = Assembly.GetExecutingAssembly() .GetManifestResourceNames() .ToList(); @@ -112,12 +113,19 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) => } + _logger.LogInformation($"Completing diagnostics channel"); Context.Collector.Channel.TryComplete(); + _logger.LogInformation($"Generating documentation compilation state"); await GenerateDocumentationState(ctx); + _logger.LogInformation($"Generating links.json"); await GenerateLinkReference(ctx); + _logger.LogInformation($"Completing diagnostics channel"); + await Context.Collector.StopAsync(ctx); + + _logger.LogInformation($"Completed diagnostics channel"); } private async Task ProcessFile(HashSet offendingFiles, DocumentationFile file, DateTimeOffset outputSeenChanges, CancellationToken token) diff --git a/src/Elastic.Markdown/IO/Discovery/GitCheckoutInformation.cs b/src/Elastic.Markdown/IO/Discovery/GitCheckoutInformation.cs index 8c0ce1f87..bfed03b3f 100644 --- a/src/Elastic.Markdown/IO/Discovery/GitCheckoutInformation.cs +++ b/src/Elastic.Markdown/IO/Discovery/GitCheckoutInformation.cs @@ -40,9 +40,9 @@ public string? RepositoryName public static GitCheckoutInformation Create(IFileSystem fileSystem) { // filesystem is not real so return a dummy + var fakeRef = Guid.NewGuid().ToString().Substring(0, 16); if (fileSystem is not FileSystem) { - var fakeRef = Guid.NewGuid().ToString().Substring(0, 16); return new GitCheckoutInformation { Branch = $"test-{fakeRef}", @@ -56,14 +56,14 @@ public static GitCheckoutInformation Create(IFileSystem fileSystem) if (!gitConfig.Exists) return Unavailable; - var head = Read(".git/HEAD"); + var head = Read(".git/HEAD") ?? fakeRef; var gitRef = head; var branch = head.Replace("refs/heads/", string.Empty); //not detached HEAD if (head.StartsWith("ref:")) { head = head.Replace("ref: ", string.Empty); - gitRef = Read(".git/" + head); + gitRef = Read(".git/" + head) ?? fakeRef; branch = branch.Replace("ref: ", string.Empty); } else @@ -94,8 +94,13 @@ public static GitCheckoutInformation Create(IFileSystem fileSystem) IFileInfo Git(string path) => fileSystem.FileInfo.New(Path.Combine(Paths.Root.FullName, path)); - string Read(string path) => - fileSystem.File.ReadAllText(Git(path).FullName).Trim(Environment.NewLine.ToCharArray()); + string? Read(string path) + { + var gitPath = Git(path).FullName; + if (!fileSystem.File.Exists(gitPath)) + return null; + return fileSystem.File.ReadAllText(gitPath).Trim(Environment.NewLine.ToCharArray()); + } string BranchTrackingRemote(string b, IniFile c) { diff --git a/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockParser.cs b/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockParser.cs index fefaaa86e..65fe02514 100644 --- a/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockParser.cs +++ b/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockParser.cs @@ -87,6 +87,8 @@ public override bool Close(BlockProcessor processor, Block block) originatingLine++; var line = lines.Lines[index]; var span = line.Slice.AsSpan(); + if (span.IndexOf("{{") < 0) + continue; if (span.ReplaceSubstitutions(context.FrontMatter?.Properties, out var replacement)) { diff --git a/src/docs-builder/Diagnostics/Console/ConsoleDiagnosticsCollector.cs b/src/docs-builder/Diagnostics/Console/ConsoleDiagnosticsCollector.cs index 6edb423da..1ac64c0d2 100644 --- a/src/docs-builder/Diagnostics/Console/ConsoleDiagnosticsCollector.cs +++ b/src/docs-builder/Diagnostics/Console/ConsoleDiagnosticsCollector.cs @@ -14,14 +14,21 @@ public class ConsoleDiagnosticsCollector(ILoggerFactory loggerFactory, ICoreServ : DiagnosticsCollector([new Log(loggerFactory.CreateLogger()), new GithubAnnotationOutput(githubActions)] ) { - private readonly List _items = new(); + private readonly List _errors = new(); + private readonly List _warnings = new(); - protected override void HandleItem(Diagnostic diagnostic) => _items.Add(diagnostic); + protected override void HandleItem(Diagnostic diagnostic) + { + if (diagnostic.Severity == Severity.Warning) + _warnings.Add(diagnostic); + else + _errors.Add(diagnostic); + } public override async Task StopAsync(Cancel ctx) { var repository = new ErrataFileSourceRepository(); - repository.WriteDiagnosticsToConsole(_items); + repository.WriteDiagnosticsToConsole(_errors, _warnings); AnsiConsole.WriteLine(); AnsiConsole.Write(new Markup($" [bold red]{Errors} Errors[/] / [bold blue]{Warnings} Warnings[/]")); diff --git a/src/docs-builder/Diagnostics/Console/ErrataFileSourceRepository.cs b/src/docs-builder/Diagnostics/Console/ErrataFileSourceRepository.cs index 5448dc18b..746aede44 100644 --- a/src/docs-builder/Diagnostics/Console/ErrataFileSourceRepository.cs +++ b/src/docs-builder/Diagnostics/Console/ErrataFileSourceRepository.cs @@ -22,10 +22,14 @@ public bool TryGet(string id, [NotNullWhen(true)] out Source? source) return true; } - public void WriteDiagnosticsToConsole(IReadOnlyCollection items) + public void WriteDiagnosticsToConsole(IReadOnlyCollection errors, IReadOnlyCollection warnings) { var report = new Report(this); - foreach (var item in items) + var limttedErrors = errors.Take(100).ToArray(); + var limittedWarnings = warnings.Take(100 - limttedErrors.Length); + var limitted = limittedWarnings.Concat(limttedErrors).ToArray(); + + foreach (var item in limitted) { var d = item.Severity switch { @@ -49,5 +53,15 @@ public void WriteDiagnosticsToConsole(IReadOnlyCollection items) // Render the report report.Render(AnsiConsole.Console); + + var totalErrorCount = errors.Count + warnings.Count; + if (limitted.Length >= totalErrorCount) + return; + + AnsiConsole.WriteLine(); + AnsiConsole.WriteLine(); + AnsiConsole.Write(new Markup($"Displayed first [bold]{limitted.Length}[/] error/warnings out of [bold]{totalErrorCount}[/]")); + + AnsiConsole.WriteLine(); } } From 517fb9c1b40b0e5e3a9f8ddecfc734a6c3aa76b8 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 15 Jan 2025 13:10:03 +0100 Subject: [PATCH 2/3] Refine callout parsing and substitution logic. Simplified code substitution checks and improved callout parsing by introducing length constraints and better inline match detection. Adjusted interpolation to skip unnecessary substitutions for efficiency. --- src/Elastic.Markdown/Helpers/Interpolation.cs | 3 +++ .../Myst/CodeBlocks/CallOutParser.cs | 2 +- .../Myst/CodeBlocks/EnhancedCodeBlockParser.cs | 18 +++++++++++++----- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/Elastic.Markdown/Helpers/Interpolation.cs b/src/Elastic.Markdown/Helpers/Interpolation.cs index 1c294847b..8ccc23156 100644 --- a/src/Elastic.Markdown/Helpers/Interpolation.cs +++ b/src/Elastic.Markdown/Helpers/Interpolation.cs @@ -17,6 +17,9 @@ public static class Interpolation public static bool ReplaceSubstitutions(this ReadOnlySpan span, Dictionary? properties, out string? replacement) { replacement = null; + if (span.IndexOf("}}") < 0) + return false; + var substitutions = properties ?? new(); if (substitutions.Count == 0) return false; diff --git a/src/Elastic.Markdown/Myst/CodeBlocks/CallOutParser.cs b/src/Elastic.Markdown/Myst/CodeBlocks/CallOutParser.cs index 645281a45..97a8a1569 100644 --- a/src/Elastic.Markdown/Myst/CodeBlocks/CallOutParser.cs +++ b/src/Elastic.Markdown/Myst/CodeBlocks/CallOutParser.cs @@ -11,6 +11,6 @@ public static partial class CallOutParser [GeneratedRegex(@"^.+\S+.*?\s<\d+>$", RegexOptions.IgnoreCase, "en-US")] public static partial Regex CallOutNumber(); - [GeneratedRegex(@"^.+\S+.*?\s(?:\/\/|#)\s[^""]+$", RegexOptions.IgnoreCase, "en-US")] + [GeneratedRegex(@"^.+\S+.*?\s(?:\/\/|#)\s[^""\/#]+$", RegexOptions.IgnoreCase, "en-US")] public static partial Regex MathInlineAnnotation(); } diff --git a/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockParser.cs b/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockParser.cs index 65fe02514..2d3f5844a 100644 --- a/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockParser.cs +++ b/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockParser.cs @@ -87,8 +87,6 @@ public override bool Close(BlockProcessor processor, Block block) originatingLine++; var line = lines.Lines[index]; var span = line.Slice.AsSpan(); - if (span.IndexOf("{{") < 0) - continue; if (span.ReplaceSubstitutions(context.FrontMatter?.Properties, out var replacement)) { @@ -100,10 +98,20 @@ public override bool Close(BlockProcessor processor, Block block) if (codeBlock.OpeningFencedCharCount > 3) continue; - var matchClassicCallout = CallOutParser.CallOutNumber().EnumerateMatches(span); - var callOut = EnumerateAnnotations(matchClassicCallout, ref span, ref callOutIndex, originatingLine, false); + if (span.IndexOf("<") < 0 && span.IndexOf("//") < 0) + continue; + + Console.WriteLine("Code subbing: " + context.Path.FullName); + CallOut? callOut = null; - if (callOut is null) + if (span.IndexOf("<") > 0) + { + var matchClassicCallout = CallOutParser.CallOutNumber().EnumerateMatches(span); + callOut = EnumerateAnnotations(matchClassicCallout, ref span, ref callOutIndex, originatingLine, false); + } + + // only support magic callouts for smaller line lengths + if (callOut is null && span.Length < 200) { var matchInline = CallOutParser.MathInlineAnnotation().EnumerateMatches(span); callOut = EnumerateAnnotations(matchInline, ref span, ref callOutIndex, originatingLine, From a5a1397fb880e3fa533ba957af7778b832186a44 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 15 Jan 2025 13:18:20 +0100 Subject: [PATCH 3/3] remove debug console writelines --- src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs | 1 - src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockParser.cs | 1 - 2 files changed, 2 deletions(-) diff --git a/src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs b/src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs index 1f1389533..d6848520d 100644 --- a/src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs +++ b/src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs @@ -124,7 +124,6 @@ protected virtual void HandleItem(Diagnostic diagnostic) { } public virtual async Task StopAsync(CancellationToken cancellationToken) { - Console.WriteLine("Stopping diagnostics"); if (_started is not null) await _started; await Channel.Reader.Completion; diff --git a/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockParser.cs b/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockParser.cs index 2d3f5844a..a7f2578a3 100644 --- a/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockParser.cs +++ b/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockParser.cs @@ -101,7 +101,6 @@ public override bool Close(BlockProcessor processor, Block block) if (span.IndexOf("<") < 0 && span.IndexOf("//") < 0) continue; - Console.WriteLine("Code subbing: " + context.Path.FullName); CallOut? callOut = null; if (span.IndexOf("<") > 0)