Add caching

Author: theletterf

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -16,6 +16,7 @@
 using Elastic.Markdown.Helpers;
 using Elastic.Markdown.IO;
 using Elastic.Markdown.Links.CrossLinks;
+using Elastic.Markdown.Myst.Directives.Diagram;
 using Elastic.Markdown.Myst.Renderers;
 using Elastic.Markdown.Myst.Renderers.LlmMarkdown;
 using Markdig.Syntax;
@@ -106,6 +107,9 @@ public async Task ResolveDirectoryTree(Cancel ctx)
 
 	public async Task<GenerationResult> GenerateAll(Cancel ctx)
 	{
+		// Clear diagram registry for fresh tracking
+		DiagramRegistry.Clear();
+
 		var result = new GenerationResult();
 
 		var generationState = Context.SkipDocumentationState ? null : GetPreviousGenerationState();
@@ -142,6 +146,13 @@ public async Task<GenerationResult> GenerateAll(Cancel ctx)
 		_logger.LogInformation($"Generating links.json");
 		var linkReference = await GenerateLinkReference(ctx);
 
+		// Clean up unused diagram files
+		var cleanedCount = DiagramRegistry.CleanupUnusedDiagrams(DocumentationSet.OutputDirectory.FullName);
+		if (cleanedCount > 0)
+		{
+			_logger.LogInformation("Cleaned up {CleanedCount} unused diagram files", cleanedCount);
+		}
+
 		// ReSharper disable once WithExpressionModifiesAllMembers
 		return result with
 		{

src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs

@@ -2,6 +2,8 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System.Security.Cryptography;
+using System.Text;
 using Elastic.Markdown.Diagnostics;
 
 namespace Elastic.Markdown.Myst.Directives.Diagram;
@@ -25,6 +27,16 @@ public class DiagramBlock(DirectiveBlockParser parser, ParserContext context) :
 	/// </summary>
 	public string? EncodedUrl { get; private set; }
 
+	/// <summary>
+	/// The local SVG path relative to the output directory
+	/// </summary>
+	public string? LocalSvgPath { get; private set; }
+
+	/// <summary>
+	/// Content hash for unique identification and caching
+	/// </summary>
+	public string? ContentHash { get; private set; }
+
 	public override void FinalizeAndValidate(ParserContext context)
 	{
 		// Extract diagram type from arguments or default to "mermaid"
@@ -39,6 +51,12 @@ public override void FinalizeAndValidate(ParserContext context)
 			return;
 		}
 
+		// Generate content hash for caching
+		ContentHash = GenerateContentHash(DiagramType, Content);
+
+		// Generate local path for cached SVG
+		LocalSvgPath = GenerateLocalPath(context);
+
 		// Generate the encoded URL for Kroki
 		try
 		{
@@ -47,7 +65,14 @@ public override void FinalizeAndValidate(ParserContext context)
 		catch (Exception ex)
 		{
 			this.EmitError($"Failed to encode diagram: {ex.Message}", ex);
+			return;
 		}
+
+		// Register diagram for tracking and cleanup
+		DiagramRegistry.RegisterDiagram(LocalSvgPath);
+
+		// Cache diagram asynchronously
+		_ = Task.Run(() => TryCacheDiagramAsync(context));
 	}
 
 	private string? ExtractContent()
@@ -68,4 +93,60 @@ public override void FinalizeAndValidate(ParserContext context)
 
 		return lines.Count > 0 ? string.Join("\n", lines) : null;
 	}
+
+	private string GenerateContentHash(string diagramType, string content)
+	{
+		var input = $"{diagramType}:{content}";
+		var bytes = Encoding.UTF8.GetBytes(input);
+		var hash = SHA256.HashData(bytes);
+		return Convert.ToHexString(hash)[..12].ToLowerInvariant();
+	}
+
+	private string GenerateLocalPath(ParserContext context)
+	{
+		var markdownFileName = "unknown";
+		if (context.MarkdownSourcePath?.FullName != null)
+		{
+			markdownFileName = Path.GetFileNameWithoutExtension(context.MarkdownSourcePath.FullName);
+		}
+
+		var filename = $"{markdownFileName}-diagram-{DiagramType}-{ContentHash}.svg";
+		return Path.Combine("images", "generated-graphs", filename);
+	}
+
+	private async Task TryCacheDiagramAsync(ParserContext context)
+	{
+		if (string.IsNullOrEmpty(EncodedUrl) || string.IsNullOrEmpty(LocalSvgPath))
+			return;
+
+		try
+		{
+			// Determine the full output path
+			var outputDirectory = context.Build.DocumentationOutputDirectory.FullName;
+			var fullPath = Path.Combine(outputDirectory, LocalSvgPath);
+
+			// Skip if file already exists
+			if (File.Exists(fullPath))
+				return;
+
+			// Create directory if it doesn't exist
+			var directory = Path.GetDirectoryName(fullPath);
+			if (directory != null && !Directory.Exists(directory))
+			{
+				_ = Directory.CreateDirectory(directory);
+			}
+
+			// Download SVG from Kroki
+			using var httpClient = new HttpClient { Timeout = TimeSpan.FromSeconds(30) };
+			var svgContent = await httpClient.GetStringAsync(EncodedUrl);
+
+			// Write to local file
+			await File.WriteAllTextAsync(fullPath, svgContent);
+		}
+		catch
+		{
+			// Silent failure - caching is opportunistic
+			// The system will fall back to Kroki URLs
+		}
+	}
 }

src/Elastic.Markdown/Myst/Directives/Diagram/DiagramRegistry.cs

@@ -0,0 +1,146 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.IO.Abstractions;
+
+namespace Elastic.Markdown.Myst.Directives.Diagram;
+
+/// <summary>
+/// Registry to track active diagrams and manage cleanup of outdated cached files
+/// </summary>
+public static class DiagramRegistry
+{
+	private static readonly HashSet<string> ActiveDiagrams = [];
+	private static readonly Lock Lock = new();
+
+	/// <summary>
+	/// Register a diagram as active during the current build
+	/// </summary>
+	/// <param name="localSvgPath">The local SVG path relative to output directory</param>
+	public static void RegisterDiagram(string localSvgPath)
+	{
+		if (string.IsNullOrEmpty(localSvgPath))
+			return;
+
+		lock (Lock)
+		{
+			_ = ActiveDiagrams.Add(localSvgPath);
+		}
+	}
+
+	/// <summary>
+	/// Get all currently registered active diagrams
+	/// </summary>
+	/// <returns>Collection of active diagram paths</returns>
+	public static IReadOnlyCollection<string> GetActiveDiagrams()
+	{
+		lock (Lock)
+		{
+			return ActiveDiagrams.ToArray();
+		}
+	}
+
+	/// <summary>
+	/// Clear all registered diagrams (typically called at start of build)
+	/// </summary>
+	public static void Clear()
+	{
+		lock (Lock)
+		{
+			ActiveDiagrams.Clear();
+		}
+	}
+
+	/// <summary>
+	/// Clean up unused diagram files from the output directory
+	/// </summary>
+	/// <param name="outputDirectory">The output directory path</param>
+	/// <returns>Number of files cleaned up</returns>
+	public static int CleanupUnusedDiagrams(string outputDirectory) =>
+		CleanupUnusedDiagrams(outputDirectory, new FileSystem());
+
+	/// <summary>
+	/// Clean up unused diagram files from the output directory
+	/// </summary>
+	/// <param name="outputDirectory">The output directory path</param>
+	/// <param name="fileSystem">File system abstraction for testing</param>
+	/// <returns>Number of files cleaned up</returns>
+	public static int CleanupUnusedDiagrams(string outputDirectory, IFileSystem fileSystem)
+	{
+		if (string.IsNullOrEmpty(outputDirectory))
+			return 0;
+
+		var graphsDir = fileSystem.Path.Combine(outputDirectory, "images", "generated-graphs");
+		if (!fileSystem.Directory.Exists(graphsDir))
+			return 0;
+
+		var cleanedCount = 0;
+		var activePaths = GetActiveDiagrams();
+
+		try
+		{
+			var existingFiles = fileSystem.Directory.GetFiles(graphsDir, "*.svg", SearchOption.AllDirectories);
+
+			foreach (var file in existingFiles)
+			{
+				var relativePath = fileSystem.Path.GetRelativePath(outputDirectory, file);
+
+				// Convert to forward slashes for consistent comparison
+				var normalizedPath = relativePath.Replace(fileSystem.Path.DirectorySeparatorChar, '/');
+
+				if (!activePaths.Any(active => active.Replace(fileSystem.Path.DirectorySeparatorChar, '/') == normalizedPath))
+				{
+					try
+					{
+						fileSystem.File.Delete(file);
+						cleanedCount++;
+					}
+					catch
+					{
+						// Silent failure - cleanup is opportunistic
+					}
+				}
+			}
+
+			// Clean up empty directories
+			CleanupEmptyDirectories(graphsDir, fileSystem);
+		}
+		catch
+		{
+			// Silent failure - cleanup is opportunistic
+		}
+
+		return cleanedCount;
+	}
+
+	/// <summary>
+	/// Remove empty directories recursively
+	/// </summary>
+	/// <param name="directory">Directory to clean up</param>
+	/// <param name="fileSystem">File system abstraction</param>
+	private static void CleanupEmptyDirectories(string directory, IFileSystem fileSystem)
+	{
+		try
+		{
+			if (!fileSystem.Directory.Exists(directory))
+				return;
+
+			// Clean up subdirectories first
+			foreach (var subDir in fileSystem.Directory.GetDirectories(directory))
+			{
+				CleanupEmptyDirectories(subDir, fileSystem);
+			}
+
+			// Remove directory if it's empty
+			if (!fileSystem.Directory.EnumerateFileSystemEntries(directory).Any())
+			{
+				fileSystem.Directory.Delete(directory);
+			}
+		}
+		catch
+		{
+			// Silent failure - cleanup is opportunistic
+		}
+	}
+}

src/Elastic.Markdown/Myst/Directives/Diagram/DiagramView.cshtml

@@ -4,7 +4,17 @@
 	if (diagram?.EncodedUrl != null)
 	{
 		<div class="diagram" data-diagram-type="@diagram.DiagramType">
-			<img src="@diagram.EncodedUrl" alt="@diagram.DiagramType diagram" loading="lazy" />
+			@if (!string.IsNullOrEmpty(diagram.LocalSvgPath))
+			{
+				<img src="@diagram.LocalSvgPath" 
+				     alt="@diagram.DiagramType diagram" 
+				     loading="lazy"
+				     onerror="this.src='@diagram.EncodedUrl'" />
+			}
+			else
+			{
+				<img src="@diagram.EncodedUrl" alt="@diagram.DiagramType diagram" loading="lazy" />
+			}
 		</div>
 	}
 	else