diff --git a/Directory.Packages.props b/Directory.Packages.props index 4a9060035..b46ee9a91 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -16,6 +16,7 @@ + diff --git a/docs/_docset.yml b/docs/_docset.yml index 8ab198efe..5f3905701 100644 --- a/docs/_docset.yml +++ b/docs/_docset.yml @@ -89,7 +89,7 @@ toc: - file: code.md - file: comments.md - file: conditionals.md - - hidden: diagrams.md + - file: diagrams.md - file: dropdowns.md - file: definition-lists.md - file: example_blocks.md diff --git a/docs/syntax/diagrams.md b/docs/syntax/diagrams.md index ce20375de..792dcf216 100644 --- a/docs/syntax/diagrams.md +++ b/docs/syntax/diagrams.md @@ -2,6 +2,10 @@ The `diagram` directive allows you to render various types of diagrams using the [Kroki](https://kroki.io/) service. Kroki supports many diagram types including Mermaid, D2, Graphviz, PlantUML, and more. +::::{warning} +This is an experimental feature. It may change in the future. +:::: + ## Basic usage The basic syntax for the diagram directive is: @@ -84,7 +88,7 @@ sequenceDiagram :::::{tab-item} Rendered ::::{diagram} mermaid sequenceDiagram - participant A as Alice + participant A as Ada participant B as Bob A->>B: Hello Bob, how are you? B-->>A: Great! diff --git a/src/Elastic.Documentation.Configuration/BuildContext.cs b/src/Elastic.Documentation.Configuration/BuildContext.cs index 154d90b79..22479014a 100644 --- a/src/Elastic.Documentation.Configuration/BuildContext.cs +++ b/src/Elastic.Documentation.Configuration/BuildContext.cs @@ -6,6 +6,7 @@ using System.Reflection; using Elastic.Documentation.Configuration.Assembler; using Elastic.Documentation.Configuration.Builder; +using Elastic.Documentation.Configuration.Diagram; using Elastic.Documentation.Configuration.Versions; using Elastic.Documentation.Diagnostics; diff --git a/src/Elastic.Documentation.Configuration/Diagram/DiagramRegistry.cs b/src/Elastic.Documentation.Configuration/Diagram/DiagramRegistry.cs new file mode 100644 index 000000000..9751083ff --- /dev/null +++ b/src/Elastic.Documentation.Configuration/Diagram/DiagramRegistry.cs @@ -0,0 +1,210 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Collections.Concurrent; +using System.IO.Abstractions; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.Extensions; +using Microsoft.Extensions.Logging; + +namespace Elastic.Documentation.Configuration.Diagram; + +/// +/// Information about a diagram that needs to be cached +/// +/// The intended cache output file location +/// Encoded Kroki URL for downloading +public record DiagramCacheInfo(IFileInfo OutputFile, string EncodedUrl); + +/// Registry to track active diagrams and manage cleanup of outdated cached files +public class DiagramRegistry(ILoggerFactory logFactory, BuildContext context) : IDisposable +{ + private readonly ILogger _logger = logFactory.CreateLogger(); + private readonly ConcurrentDictionary _diagramsToCache = new(); + private readonly IFileSystem _writeFileSystem = context.WriteFileSystem; + private readonly IFileSystem _readFileSystem = context.ReadFileSystem; + private readonly HttpClient _httpClient = new() { Timeout = TimeSpan.FromSeconds(30) }; + + /// + /// Register a diagram for caching (collects info for later batch processing) + /// + /// The local SVG path relative to the output directory + /// The encoded Kroki URL for downloading + /// The full path to the output directory + public void RegisterDiagramForCaching(IFileInfo outputFile, string encodedUrl) + { + if (string.IsNullOrEmpty(encodedUrl)) + return; + + if (!outputFile.IsSubPathOf(context.DocumentationOutputDirectory)) + return; + + _ = _diagramsToCache.TryAdd(outputFile.FullName, new DiagramCacheInfo(outputFile, encodedUrl)); + } + + /// + /// Create cached diagram files by downloading from Kroki in parallel + /// + /// Number of diagrams downloaded + public async Task CreateDiagramCachedFiles(Cancel ctx) + { + if (_diagramsToCache.IsEmpty) + return 0; + + var downloadCount = 0; + + await Parallel.ForEachAsync(_diagramsToCache.Values, new ParallelOptions + { + MaxDegreeOfParallelism = Environment.ProcessorCount, + CancellationToken = ctx + }, async (diagramInfo, ct) => + { + var localPath = _readFileSystem.Path.GetRelativePath(context.DocumentationOutputDirectory.FullName, diagramInfo.OutputFile.FullName); + + try + { + if (!diagramInfo.OutputFile.IsSubPathOf(context.DocumentationOutputDirectory)) + return; + + // Skip if the file already exists + if (_readFileSystem.File.Exists(diagramInfo.OutputFile.FullName)) + return; + + // If we are running on CI, and we are creating cached files we should fail the build and alert the user to create them + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("CI"))) + { + context.Collector.EmitGlobalError($"Discovered new diagram SVG '{localPath}' please run `docs-builder --force` to ensure a cached version is generated"); + return; + } + + // Create the directory if needed + var directory = _writeFileSystem.Path.GetDirectoryName(diagramInfo.OutputFile.FullName); + if (directory != null && !_writeFileSystem.Directory.Exists(directory)) + _ = _writeFileSystem.Directory.CreateDirectory(directory); + + _logger.LogWarning("Creating local diagram: {LocalPath}", localPath); + // Download SVG content + var svgContent = await _httpClient.GetStringAsync(diagramInfo.EncodedUrl, ct); + + // Validate SVG content + if (string.IsNullOrWhiteSpace(svgContent) || !svgContent.Contains(" 0) + _logger.LogInformation("Downloaded {DownloadCount} diagram files from Kroki", downloadCount); + + return downloadCount; + } + + /// + /// Clean up unused diagram files from the cache directory + /// + /// Number of files cleaned up + public int CleanupUnusedDiagrams() + { + if (!_readFileSystem.Directory.Exists(context.DocumentationOutputDirectory.FullName)) + return 0; + var folders = _writeFileSystem.Directory.GetDirectories(context.DocumentationOutputDirectory.FullName, "generated-graphs", SearchOption.AllDirectories); + var existingFiles = folders + .Select(f => (Folder: f, Files: _writeFileSystem.Directory.GetFiles(f, "*.svg", SearchOption.TopDirectoryOnly))) + .ToArray(); + if (existingFiles.Length == 0) + return 0; + var cleanedCount = 0; + + try + { + foreach (var (folder, files) in existingFiles) + { + foreach (var file in files) + { + if (_diagramsToCache.ContainsKey(file)) + continue; + try + { + _writeFileSystem.File.Delete(file); + cleanedCount++; + } + catch + { + // Silent failure - cleanup is opportunistic + } + } + // Clean up empty directories + CleanupEmptyDirectories(folder); + } + } + catch + { + // Silent failure - cleanup is opportunistic + } + + return cleanedCount; + } + + private void CleanupEmptyDirectories(string directory) + { + try + { + var folder = _writeFileSystem.DirectoryInfo.New(directory); + if (!folder.IsSubPathOf(context.DocumentationOutputDirectory)) + return; + + if (folder.Name != "generated-graphs") + return; + + if (_writeFileSystem.Directory.EnumerateFileSystemEntries(folder.FullName).Any()) + return; + + _writeFileSystem.Directory.Delete(folder.FullName); + + var parentFolder = folder.Parent; + if (parentFolder is null || parentFolder.Name != "images") + return; + + if (_writeFileSystem.Directory.EnumerateFileSystemEntries(parentFolder.FullName).Any()) + return; + + _writeFileSystem.Directory.Delete(folder.FullName); + } + catch + { + // Silent failure - cleanup is opportunistic + } + } + + /// + /// Dispose of resources, including the HttpClient + /// + public void Dispose() + { + _httpClient.Dispose(); + GC.SuppressFinalize(this); + } +} diff --git a/src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs b/src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs index fb1d24f06..86cb66b9e 100644 --- a/src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs +++ b/src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs @@ -53,5 +53,3 @@ public static void EmitGlobalWarning(this IDiagnosticsCollector collector, strin public static void EmitGlobalHint(this IDiagnosticsCollector collector, string message) => collector.EmitHint(string.Empty, message); } - - diff --git a/src/Elastic.Documentation/Extensions/IFileInfoExtensions.cs b/src/Elastic.Documentation/Extensions/IFileInfoExtensions.cs index 2a80ef6cf..7534030dc 100644 --- a/src/Elastic.Documentation/Extensions/IFileInfoExtensions.cs +++ b/src/Elastic.Documentation/Extensions/IFileInfoExtensions.cs @@ -18,4 +18,29 @@ public static string ReadToEnd(this IFileInfo fileInfo) using var reader = new StreamReader(stream); return reader.ReadToEnd(); } + + /// Validates is in a subdirectory of + public static bool IsSubPathOf(this IFileInfo file, IDirectoryInfo parentDirectory) + { + var parent = file.Directory; + return parent is not null && parent.IsSubPathOf(parentDirectory); + } +} + +public static class IDirectoryInfoExtensions +{ + /// Validates is subdirectory of + public static bool IsSubPathOf(this IDirectoryInfo directory, IDirectoryInfo parentDirectory) + { + var parent = directory; + do + { + if (parent.FullName == parentDirectory.FullName) + return true; + parent = parent.Parent; + } + while (parent != null); + + return false; + } } diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs index 253dee397..a15a85d88 100644 --- a/src/Elastic.Markdown/DocumentationGenerator.cs +++ b/src/Elastic.Markdown/DocumentationGenerator.cs @@ -6,6 +6,7 @@ using System.Text.Json; using Elastic.Documentation; using Elastic.Documentation.Configuration; +using Elastic.Documentation.Configuration.Diagram; using Elastic.Documentation.Legacy; using Elastic.Documentation.Links; using Elastic.Documentation.Serialization; @@ -16,6 +17,7 @@ using Elastic.Markdown.Helpers; using Elastic.Markdown.IO; using Elastic.Markdown.Links.CrossLinks; +using Elastic.Markdown.Myst.Directives.Diagram; using Elastic.Markdown.Myst.Renderers; using Elastic.Markdown.Myst.Renderers.LlmMarkdown; using Markdig.Syntax; @@ -142,6 +144,9 @@ public async Task GenerateAll(Cancel ctx) _logger.LogInformation($"Generating links.json"); var linkReference = await GenerateLinkReference(ctx); + await CreateDiagramCachedFiles(ctx); + CleanupUnusedDiagrams(); + // ReSharper disable once WithExpressionModifiesAllMembers return result with { @@ -149,6 +154,26 @@ public async Task GenerateAll(Cancel ctx) }; } + /// + /// Downloads diagram files in parallel from Kroki + /// + public async Task CreateDiagramCachedFiles(Cancel ctx) + { + var downloadedCount = await DocumentationSet.DiagramRegistry.CreateDiagramCachedFiles(ctx); + if (downloadedCount > 0) + _logger.LogInformation("Downloaded {DownloadedCount} diagram files from Kroki", downloadedCount); + } + + /// + /// Cleans up unused diagram files from the output directory + /// + public void CleanupUnusedDiagrams() + { + var cleanedCount = DocumentationSet.DiagramRegistry.CleanupUnusedDiagrams(); + if (cleanedCount > 0) + _logger.LogInformation("Cleaned up {CleanedCount} unused diagram files", cleanedCount); + } + private async Task ProcessDocumentationFiles(HashSet offendingFiles, DateTimeOffset outputSeenChanges, Cancel ctx) { var processedFileCount = 0; diff --git a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs index df2a9fb57..f1be56972 100644 --- a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs +++ b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs @@ -29,14 +29,14 @@ protected override Task GetMinimalParseDocumentAsync(Cancel ct { Title = "Prebuilt detection rules reference"; var markdown = GetMarkdown(); - var document = MarkdownParser.MinimalParseStringAsync(markdown, SourceFile, null); + var document = MarkdownParser.MinimalParseString(markdown, SourceFile, null); return Task.FromResult(document); } protected override Task GetParseDocumentAsync(Cancel ctx) { var markdown = GetMarkdown(); - var document = MarkdownParser.ParseStringAsync(markdown, SourceFile, null); + var document = MarkdownParser.ParseString(markdown, SourceFile, null); return Task.FromResult(document); } @@ -127,14 +127,14 @@ protected override Task GetMinimalParseDocumentAsync(Cancel ct { Title = Rule?.Name; var markdown = GetMarkdown(); - var document = MarkdownParser.MinimalParseStringAsync(markdown, RuleSourceMarkdownPath, null); + var document = MarkdownParser.MinimalParseString(markdown, RuleSourceMarkdownPath, null); return Task.FromResult(document); } protected override Task GetParseDocumentAsync(Cancel ctx) { var markdown = GetMarkdown(); - var document = MarkdownParser.ParseStringAsync(markdown, RuleSourceMarkdownPath, null); + var document = MarkdownParser.ParseString(markdown, RuleSourceMarkdownPath, null); return Task.FromResult(document); } diff --git a/src/Elastic.Markdown/HtmlWriter.cs b/src/Elastic.Markdown/HtmlWriter.cs index cdba6bcd0..0a99e74e7 100644 --- a/src/Elastic.Markdown/HtmlWriter.cs +++ b/src/Elastic.Markdown/HtmlWriter.cs @@ -43,7 +43,7 @@ public class HtmlWriter( public string Render(string markdown, IFileInfo? source) { source ??= DocumentationSet.Context.ConfigurationPath; - var parsed = DocumentationSet.MarkdownParser.ParseStringAsync(markdown, source, null); + var parsed = DocumentationSet.MarkdownParser.ParseString(markdown, source, null); return MarkdownFile.CreateHtml(parsed); } diff --git a/src/Elastic.Markdown/IO/DocumentationSet.cs b/src/Elastic.Markdown/IO/DocumentationSet.cs index 5dc9886c4..5029c7984 100644 --- a/src/Elastic.Markdown/IO/DocumentationSet.cs +++ b/src/Elastic.Markdown/IO/DocumentationSet.cs @@ -9,6 +9,7 @@ using Elastic.Documentation; using Elastic.Documentation.Configuration; using Elastic.Documentation.Configuration.Builder; +using Elastic.Documentation.Configuration.Diagram; using Elastic.Documentation.Configuration.TableOfContents; using Elastic.Documentation.LinkIndex; using Elastic.Documentation.Links; @@ -132,6 +133,8 @@ public class DocumentationSet : INavigationLookups, IPositionalNavigation public ConcurrentDictionary NavigationRenderResults { get; } = []; + public DiagramRegistry DiagramRegistry { get; } + public DocumentationSet( BuildContext context, ILoggerFactory logFactory, @@ -143,6 +146,7 @@ public DocumentationSet( Source = ContentSourceMoniker.Create(context.Git.RepositoryName, null); SourceDirectory = context.DocumentationSourceDirectory; OutputDirectory = context.DocumentationOutputDirectory; + DiagramRegistry = new DiagramRegistry(logFactory, context); LinkResolver = linkResolver ?? new CrossLinkResolver(new ConfigurationCrossLinkFetcher(logFactory, context.Configuration, Aws3LinkIndexReader.CreateAnonymous())); Configuration = context.Configuration; @@ -154,7 +158,7 @@ public DocumentationSet( CrossLinkResolver = LinkResolver, DocumentationFileLookup = DocumentationFileLookup }; - MarkdownParser = new MarkdownParser(context, resolver); + MarkdownParser = new MarkdownParser(context, resolver, DiagramRegistry); Name = Context.Git != GitCheckoutInformation.Unavailable ? Context.Git.RepositoryName diff --git a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs index f9a7c75fb..6feb6e7d1 100644 --- a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs +++ b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs @@ -2,7 +2,11 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using System.Security.Cryptography; +using System.Text; +using Elastic.Documentation.Configuration.Diagram; using Elastic.Markdown.Diagnostics; +using Elastic.Markdown.IO; namespace Elastic.Markdown.Myst.Directives.Diagram; @@ -25,6 +29,14 @@ public class DiagramBlock(DirectiveBlockParser parser, ParserContext context) : /// public string? EncodedUrl { get; private set; } + /// The local SVG Url + public string? LocalSvgUrl { get; private set; } + + /// + /// Content hash for unique identification and caching + /// + public string? ContentHash { get; private set; } + public override void FinalizeAndValidate(ParserContext context) { // Extract diagram type from arguments or default to "mermaid" @@ -39,6 +51,13 @@ public override void FinalizeAndValidate(ParserContext context) return; } + // Generate content hash for caching + ContentHash = GenerateContentHash(DiagramType, Content); + + // Generate the local path and url for cached SVG + var localPath = GenerateLocalPath(context); + LocalSvgUrl = localPath.Replace(Path.DirectorySeparatorChar, '/'); + // Generate the encoded URL for Kroki try { @@ -47,7 +66,20 @@ public override void FinalizeAndValidate(ParserContext context) catch (Exception ex) { this.EmitError($"Failed to encode diagram: {ex.Message}", ex); + return; + } + + // only register SVG if we can look up the Markdown + if (context.DocumentationFileLookup(context.MarkdownSourcePath) is MarkdownFile currentMarkdown) + { + var fs = context.Build.ReadFileSystem; + var scopePath = fs.FileInfo.New(Path.Combine(currentMarkdown.ScopeDirectory.FullName, localPath)); + var relativeScopePath = fs.Path.GetRelativePath(context.Build.DocumentationSourceDirectory.FullName, scopePath.FullName); + var outputPath = fs.FileInfo.New(Path.Combine(context.Build.DocumentationOutputDirectory.FullName, relativeScopePath)); + context.DiagramRegistry.RegisterDiagramForCaching(outputPath, EncodedUrl); } + else + this.EmitError($"Can not locate markdown source for {context.MarkdownSourcePath} to register diagram for caching."); } private string? ExtractContent() @@ -68,4 +100,25 @@ public override void FinalizeAndValidate(ParserContext context) return lines.Count > 0 ? string.Join("\n", lines) : null; } + + private string GenerateContentHash(string diagramType, string content) + { + var input = $"{diagramType}:{content}"; + var bytes = Encoding.UTF8.GetBytes(input); + var hash = SHA256.HashData(bytes); + return Convert.ToHexString(hash)[..12].ToLowerInvariant(); + } + + private string GenerateLocalPath(ParserContext context) + { + var markdownFileName = Path.GetFileNameWithoutExtension(context.MarkdownSourcePath.Name); + + var filename = $"{markdownFileName}-diagram-{DiagramType}-{ContentHash}.svg"; + var localPath = Path.Combine("images", "generated-graphs", filename); + + // Normalize path separators to forward slashes for web compatibility + return localPath; + } + + } diff --git a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramView.cshtml b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramView.cshtml index 40c13b2c3..e9a50d539 100644 --- a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramView.cshtml +++ b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramView.cshtml @@ -4,7 +4,17 @@ if (diagram?.EncodedUrl != null) {
- @diagram.DiagramType diagram + @if (!string.IsNullOrEmpty(diagram.LocalSvgUrl)) + { + @diagram.DiagramType diagram + } + else + { + @diagram.DiagramType diagram + }
} else diff --git a/src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs b/src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs index 4c8a6e2d6..4a04c6f86 100644 --- a/src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs +++ b/src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs @@ -294,8 +294,16 @@ private static void WriteIncludeBlock(HtmlRenderer renderer, IncludeBlock block) var snippet = block.Build.ReadFileSystem.FileInfo.New(block.IncludePath); var parentPath = block.Context.MarkdownParentPath ?? block.Context.MarkdownSourcePath; - var document = MarkdownParser.ParseSnippetAsync(block.Build, block.Context, snippet, parentPath, block.Context.YamlFrontMatter, default) - .GetAwaiter().GetResult(); + var state = new ParserState(block.Build) + { + MarkdownSourcePath = snippet, + YamlFrontMatter = block.Context.YamlFrontMatter, + DocumentationFileLookup = block.Context.DocumentationFileLookup, + CrossLinkResolver = block.Context.CrossLinkResolver, + ParentMarkdownPath = parentPath, + DiagramRegistry = block.Context.DiagramRegistry + }; + var document = MarkdownParser.ParseSnippetAsync(snippet, state, Cancel.None).GetAwaiter().GetResult(); var html = document.ToHtml(MarkdownParser.Pipeline); _ = renderer.Write(html); @@ -330,7 +338,15 @@ private static void WriteSettingsBlock(HtmlRenderer renderer, SettingsBlock bloc SettingsCollection = settings, RenderMarkdown = s => { - var document = MarkdownParser.ParseMarkdownStringAsync(block.Build, block.Context, s, block.IncludeFrom, block.Context.YamlFrontMatter, MarkdownParser.Pipeline); + var state = new ParserState(block.Build) + { + MarkdownSourcePath = block.IncludeFrom, + YamlFrontMatter = block.Context.YamlFrontMatter, + DocumentationFileLookup = block.Context.DocumentationFileLookup, + CrossLinkResolver = block.Context.CrossLinkResolver, + DiagramRegistry = block.Context.DiagramRegistry + }; + var document = MarkdownParser.ParseMarkdownString(s, MarkdownParser.Pipeline, state); var html = document.ToHtml(MarkdownParser.Pipeline); return html; } diff --git a/src/Elastic.Markdown/Myst/MarkdownParser.cs b/src/Elastic.Markdown/Myst/MarkdownParser.cs index f5e01bd21..7272591c7 100644 --- a/src/Elastic.Markdown/Myst/MarkdownParser.cs +++ b/src/Elastic.Markdown/Myst/MarkdownParser.cs @@ -5,6 +5,7 @@ using System.IO.Abstractions; using Cysharp.IO; using Elastic.Documentation.Configuration; +using Elastic.Documentation.Configuration.Diagram; using Elastic.Markdown.Helpers; using Elastic.Markdown.Myst.CodeBlocks; using Elastic.Markdown.Myst.Comments; @@ -24,7 +25,7 @@ namespace Elastic.Markdown.Myst; -public partial class MarkdownParser(BuildContext build, IParserResolvers resolvers) +public partial class MarkdownParser(BuildContext build, IParserResolvers resolvers, DiagramRegistry diagramRegistry) { private BuildContext Build { get; } = build; public IParserResolvers Resolvers { get; } = resolvers; @@ -45,30 +46,37 @@ private Task ParseFromFile( YamlFrontMatter = matter, DocumentationFileLookup = Resolvers.DocumentationFileLookup, CrossLinkResolver = Resolvers.CrossLinkResolver, - SkipValidation = skip + SkipValidation = skip, + DiagramRegistry = diagramRegistry }; var context = new ParserContext(state); return ParseAsync(path, context, pipeline, ctx); } - public MarkdownDocument ParseStringAsync(string markdown, IFileInfo path, YamlFrontMatter? matter) => - ParseMarkdownStringAsync(markdown, path, matter, Pipeline); + public MarkdownDocument ParseString(string markdown, IFileInfo path, YamlFrontMatter? matter) => + ParseMarkdownString(markdown, path, matter, Pipeline); - public MarkdownDocument MinimalParseStringAsync(string markdown, IFileInfo path, YamlFrontMatter? matter) => - ParseMarkdownStringAsync(markdown, path, matter, MinimalPipeline); + public MarkdownDocument MinimalParseString(string markdown, IFileInfo path, YamlFrontMatter? matter) => + ParseMarkdownString(markdown, path, matter, MinimalPipeline); - private MarkdownDocument ParseMarkdownStringAsync(string markdown, IFileInfo path, YamlFrontMatter? matter, MarkdownPipeline pipeline) => - ParseMarkdownStringAsync(Build, Resolvers, markdown, path, matter, pipeline); + private MarkdownDocument ParseMarkdownString(string markdown, IFileInfo path, YamlFrontMatter? matter, MarkdownPipeline pipeline) => + ParseMarkdownString(Build, Resolvers, markdown, path, matter, pipeline); - public static MarkdownDocument ParseMarkdownStringAsync(BuildContext build, IParserResolvers resolvers, string markdown, IFileInfo path, YamlFrontMatter? matter, MarkdownPipeline pipeline) + public MarkdownDocument ParseMarkdownString(BuildContext build, IParserResolvers resolvers, string markdown, IFileInfo path, YamlFrontMatter? matter, MarkdownPipeline pipeline) { var state = new ParserState(build) { MarkdownSourcePath = path, YamlFrontMatter = matter, DocumentationFileLookup = resolvers.DocumentationFileLookup, - CrossLinkResolver = resolvers.CrossLinkResolver + CrossLinkResolver = resolvers.CrossLinkResolver, + DiagramRegistry = diagramRegistry }; + return ParseMarkdownString(markdown, pipeline, state); + } + + public static MarkdownDocument ParseMarkdownString(string markdown, MarkdownPipeline pipeline, ParserState state) + { var context = new ParserContext(state); // Preprocess substitutions in link patterns before Markdig parsing @@ -78,27 +86,13 @@ public static MarkdownDocument ParseMarkdownStringAsync(BuildContext build, IPar return markdownDocument; } - public static Task ParseSnippetAsync(BuildContext build, IParserResolvers resolvers, IFileInfo path, IFileInfo parentPath, - YamlFrontMatter? matter, Cancel ctx) + public static Task ParseSnippetAsync(IFileInfo path, ParserState state, Cancel ctx) { - var state = new ParserState(build) - { - MarkdownSourcePath = path, - YamlFrontMatter = matter, - DocumentationFileLookup = resolvers.DocumentationFileLookup, - CrossLinkResolver = resolvers.CrossLinkResolver, - ParentMarkdownPath = parentPath - }; var context = new ParserContext(state); return ParseAsync(path, context, Pipeline, ctx); } - - private static async Task ParseAsync( - IFileInfo path, - MarkdownParserContext context, - MarkdownPipeline pipeline, - Cancel ctx) + private static async Task ParseAsync(IFileInfo path, MarkdownParserContext context, MarkdownPipeline pipeline, Cancel ctx) { string inputMarkdown; if (path.FileSystem is FileSystem) diff --git a/src/Elastic.Markdown/Myst/ParserContext.cs b/src/Elastic.Markdown/Myst/ParserContext.cs index 03fc703ad..7ad12c159 100644 --- a/src/Elastic.Markdown/Myst/ParserContext.cs +++ b/src/Elastic.Markdown/Myst/ParserContext.cs @@ -5,6 +5,7 @@ using System.IO.Abstractions; using Elastic.Documentation.Configuration; using Elastic.Documentation.Configuration.Builder; +using Elastic.Documentation.Configuration.Diagram; using Elastic.Markdown.Diagnostics; using Elastic.Markdown.IO; using Elastic.Markdown.Links.CrossLinks; @@ -44,6 +45,7 @@ public record ParserState(BuildContext Build) : ParserResolvers public required IFileInfo MarkdownSourcePath { get; init; } public required YamlFrontMatter? YamlFrontMatter { get; init; } + public required DiagramRegistry DiagramRegistry { get; init; } public IFileInfo? ParentMarkdownPath { get; init; } public bool SkipValidation { get; init; } @@ -58,6 +60,7 @@ public class ParserContext : MarkdownParserContext, IParserResolvers public string CurrentUrlPath { get; } public YamlFrontMatter? YamlFrontMatter { get; } public BuildContext Build { get; } + public DiagramRegistry DiagramRegistry { get; } public bool SkipValidation { get; } public Func DocumentationFileLookup { get; } public IReadOnlyDictionary Substitutions { get; } @@ -74,6 +77,7 @@ public ParserContext(ParserState state) CrossLinkResolver = state.CrossLinkResolver; MarkdownSourcePath = state.MarkdownSourcePath; DocumentationFileLookup = state.DocumentationFileLookup; + DiagramRegistry = state.DiagramRegistry; CurrentUrlPath = DocumentationFileLookup(state.ParentMarkdownPath ?? MarkdownSourcePath) is MarkdownFile md ? md.Url diff --git a/src/Elastic.Markdown/Myst/Renderers/LlmMarkdown/LlmBlockRenderers.cs b/src/Elastic.Markdown/Myst/Renderers/LlmMarkdown/LlmBlockRenderers.cs index 8fdebf0e6..b7215f45b 100644 --- a/src/Elastic.Markdown/Myst/Renderers/LlmMarkdown/LlmBlockRenderers.cs +++ b/src/Elastic.Markdown/Myst/Renderers/LlmMarkdown/LlmBlockRenderers.cs @@ -465,8 +465,16 @@ private void WriteIncludeBlock(LlmMarkdownRenderer renderer, IncludeBlock block) try { var parentPath = block.Context.MarkdownParentPath ?? block.Context.MarkdownSourcePath; - var document = MarkdownParser.ParseSnippetAsync(block.Build, block.Context, snippet, parentPath, block.Context.YamlFrontMatter, Cancel.None) - .GetAwaiter().GetResult(); + var state = new ParserState(block.Build) + { + MarkdownSourcePath = snippet, + YamlFrontMatter = block.Context.YamlFrontMatter, + DocumentationFileLookup = block.Context.DocumentationFileLookup, + CrossLinkResolver = block.Context.CrossLinkResolver, + ParentMarkdownPath = parentPath, + DiagramRegistry = block.Context.DiagramRegistry + }; + var document = MarkdownParser.ParseSnippetAsync(snippet, state, Cancel.None).GetAwaiter().GetResult(); _ = renderer.Render(document); } catch (Exception ex) diff --git a/src/tooling/docs-builder/Http/DocumentationWebHost.cs b/src/tooling/docs-builder/Http/DocumentationWebHost.cs index 8469d1431..a18c2b5a5 100644 --- a/src/tooling/docs-builder/Http/DocumentationWebHost.cs +++ b/src/tooling/docs-builder/Http/DocumentationWebHost.cs @@ -2,6 +2,7 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using System.IO; using System.IO.Abstractions; using System.Net; using System.Runtime.InteropServices; @@ -16,6 +17,7 @@ using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.FileProviders; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Westwind.AspNetCore.LiveReload; @@ -142,7 +144,7 @@ private static async Task ServeDocumentationFile(ReloadableGeneratorSta var generator = holder.Generator; const string navPartialSuffix = ".nav.html"; - // Check if the original request is asking for LLM-rendered markdown + // Check if the original request is asking for LLM-rendered Markdown var requestLlmMarkdown = slug.EndsWith(".md"); // If requesting .md output, remove the .md extension to find the source file @@ -196,6 +198,14 @@ private static async Task ServeDocumentationFile(ReloadableGeneratorSta if (s == "index.md") return Results.Redirect(generator.DocumentationSet.MarkdownFiles.First().Url); + // Check for cached SVG files (e.g., generated diagrams) in the output directory + if (Path.GetExtension(slug).Equals(".svg", StringComparison.OrdinalIgnoreCase)) + { + var svgPath = Path.Combine(generator.DocumentationSet.OutputDirectory.FullName, slug.TrimStart('/')); + if (File.Exists(svgPath)) + return Results.File(svgPath, "image/svg+xml"); + } + if (!generator.DocumentationSet.FlatMappedFiles.TryGetValue("404.md", out var notFoundDocumentationFile)) return Results.NotFound(); diff --git a/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs b/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs index b55d5eeec..020af6abc 100644 --- a/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs +++ b/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs @@ -2,6 +2,12 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using System.IO.Abstractions.TestingHelpers; +using Elastic.Documentation; +using Elastic.Documentation.Configuration; +using Elastic.Documentation.Configuration.Diagram; +using Elastic.Documentation.Configuration.Versions; +using Elastic.Documentation.Diagnostics; using Elastic.Markdown.Myst.Directives.Diagram; using FluentAssertions; @@ -30,7 +36,22 @@ flowchart LR public void GeneratesEncodedUrl() => Block!.EncodedUrl.Should().StartWith("https://kroki.io/mermaid/svg/"); [Fact] - public void RendersImageTag() => Html.Should().Contain(" Html.Should().Contain(" Block!.ContentHash.Should().NotBeNullOrEmpty(); + + [Fact] + public void GeneratesLocalSvgUrl() => Block!.LocalSvgUrl.Should().Contain("images/generated-graphs/"); + + [Fact] + public void LocalSvgPathContainsHash() => Block!.LocalSvgUrl.Should().Contain(Block!.ContentHash!); + + [Fact] + public void LocalSvgPathContainsDiagramType() => Block!.LocalSvgUrl.Should().Contain("-diagram-mermaid-"); + + [Fact] + public void RendersLocalPathWithFallback() => Html.Should().Contain("onerror=\"this.src='https://kroki.io/mermaid/svg/"); } public class DiagramBlockD2Tests(ITestOutputHelper output) : DirectiveTest(output, @@ -82,3 +103,72 @@ public class DiagramBlockEmptyTests(ITestOutputHelper output) : DirectiveTest Collector.Diagnostics.Should().ContainSingle(d => d.Message.Contains("Diagram directive requires content")); } + +public class DiagramRegistryTests +{ + private MockFileSystem FileSystem { get; } + + private BuildContext Context { get; } + + private DiagramRegistry Registry { get; } + + public DiagramRegistryTests(ITestOutputHelper output) + { + var collector = new DiagnosticsCollector([]); + var versionsConfig = new VersionsConfiguration + { + VersioningSystems = new Dictionary() + }; + FileSystem = new MockFileSystem(new Dictionary + { + { "docs/index.md", new MockFileData($"# {nameof(DiagramRegistryTests)}") } + }, new MockFileSystemOptions + { + CurrentDirectory = Paths.WorkingDirectoryRoot.FullName + }); + var root = FileSystem.DirectoryInfo.New(Path.Combine(Paths.WorkingDirectoryRoot.FullName, "docs/")); + FileSystem.GenerateDocSetYaml(root); + Context = new BuildContext(collector, FileSystem, versionsConfig); + Registry = new DiagramRegistry(new TestLoggerFactory(output), Context); + } + + [Fact] + public void CleanupUnusedDiagramsWithActiveAndUnusedFilesCleansOnlyUnused() + { + var localOutput = FileSystem.DirectoryInfo.New(Path.Combine(Context.DocumentationOutputDirectory.FullName, "output")); + var file = FileSystem.FileInfo.New(Path.Combine(localOutput.FullName, "images", "generated-graphs", "active-diagram.svg")); + Registry.RegisterDiagramForCaching(file, "http://example.com/active"); + + FileSystem.AddDirectory(Path.Combine(localOutput.FullName, "images/generated-graphs")); + FileSystem.AddFile(Path.Combine(localOutput.FullName, "images/generated-graphs/active-diagram.svg"), "active content"); + FileSystem.AddFile(Path.Combine(localOutput.FullName, "images/generated-graphs/unused-diagram.svg"), "unused content"); + + var cleanedCount = Registry.CleanupUnusedDiagrams(); + + cleanedCount.Should().Be(1); + FileSystem.File.Exists(Path.Combine(localOutput.FullName, "images/generated-graphs/active-diagram.svg")).Should().BeTrue(); + FileSystem.File.Exists(Path.Combine(localOutput.FullName, "images/generated-graphs/unused-diagram.svg")).Should().BeFalse(); + } + + [Fact] + public void CleanupUnusedDiagramsWithNonexistentDirectoryReturnsZero() + { + var cleanedCount = Registry.CleanupUnusedDiagrams(); + cleanedCount.Should().Be(0); + } + + [Fact] + public void CleanupUnusedDiagramsRemovesEmptyDirectories() + { + var localOutput = FileSystem.DirectoryInfo.New(Path.Combine(Context.DocumentationOutputDirectory.FullName, "output")); + var file = FileSystem.FileInfo.New(Path.Combine(localOutput.FullName, "images", "generated-graphs", "unused.svg")); + + FileSystem.AddDirectory(file.Directory!.FullName); + FileSystem.AddFile(file.FullName, "content"); + + var cleanedCount = Registry.CleanupUnusedDiagrams(); + + cleanedCount.Should().Be(1); + FileSystem.Directory.Exists(file.Directory.FullName).Should().BeFalse(); + } +} diff --git a/tests/Elastic.Markdown.Tests/Elastic.Markdown.Tests.csproj b/tests/Elastic.Markdown.Tests/Elastic.Markdown.Tests.csproj index 5530dffce..189720f32 100644 --- a/tests/Elastic.Markdown.Tests/Elastic.Markdown.Tests.csproj +++ b/tests/Elastic.Markdown.Tests/Elastic.Markdown.Tests.csproj @@ -15,6 +15,7 @@ +