Move registry to documentation set

Author: Mpdreamz

Directory.Packages.props

@@ -16,6 +16,7 @@
     <PackageVersion Include="AWSSDK.Core" Version="4.0.0.2" />
     <PackageVersion Include="AWSSDK.SQS" Version="4.0.0.1" />
     <PackageVersion Include="AWSSDK.S3" Version="4.0.0.1" />
+    <PackageVersion Include="Fake.Core.Context" Version="6.1.3" />
     <PackageVersion Include="FakeItEasy" Version="8.3.0" />
     <PackageVersion Include="Elastic.Ingest.Elasticsearch" Version="0.11.3" />
     <PackageVersion Include="Microsoft.OpenApi" Version="2.0.0-preview9" />

src/Elastic.Documentation.Configuration/BuildContext.cs

@@ -65,8 +65,6 @@ public string? UrlPathPrefix
 		init => _urlPathPrefix = value;
 	}
 
-	public DiagramRegistry DiagramRegistry { get; }
-
 	public BuildContext(IDiagnosticsCollector collector, IFileSystem fileSystem, VersionsConfiguration versionsConfig)
 		: this(collector, fileSystem, fileSystem, versionsConfig, null, null)
 	{
@@ -108,6 +106,5 @@ public BuildContext(
 		{
 			Enabled = false
 		};
-		DiagramRegistry = new DiagramRegistry(writeFileSystem);
 	}
 }

src/Elastic.Documentation.Configuration/Diagram/DiagramRegistry.cs

@@ -4,60 +4,49 @@
 
 using System.Collections.Concurrent;
 using System.IO.Abstractions;
+using Elastic.Documentation.Extensions;
 using Microsoft.Extensions.Logging;
 
 namespace Elastic.Documentation.Configuration.Diagram;
 
 /// <summary>
 /// Information about a diagram that needs to be cached
 /// </summary>
-/// <param name="LocalSvgPath">Local SVG path relative to output directory</param>
+/// <param name="OutputFile">The intended cache output file location</param>
 /// <param name="EncodedUrl">Encoded Kroki URL for downloading</param>
-/// <param name="OutputDirectory">Full path to output directory</param>
-public record DiagramCacheInfo(string LocalSvgPath, string EncodedUrl, string OutputDirectory);
+public record DiagramCacheInfo(IFileInfo OutputFile, string EncodedUrl);
 
-/// <summary>
 /// Registry to track active diagrams and manage cleanup of outdated cached files
-/// </summary>
-/// <param name="writeFileSystem">File system for write/delete operations during cleanup</param>
-public class DiagramRegistry(IFileSystem writeFileSystem) : IDisposable
+public class DiagramRegistry(ILoggerFactory logFactory, BuildContext context) : IDisposable
 {
-	private readonly ConcurrentDictionary<string, bool> _activeDiagrams = new();
+	private readonly ILogger<DiagramRegistry> _logger = logFactory.CreateLogger<DiagramRegistry>();
 	private readonly ConcurrentDictionary<string, DiagramCacheInfo> _diagramsToCache = new();
-	private readonly IFileSystem _writeFileSystem = writeFileSystem;
+	private readonly IFileSystem _writeFileSystem = context.WriteFileSystem;
+	private readonly IFileSystem _readFileSystem = context.ReadFileSystem;
 	private readonly HttpClient _httpClient = new() { Timeout = TimeSpan.FromSeconds(30) };
 
 	/// <summary>
 	/// Register a diagram for caching (collects info for later batch processing)
 	/// </summary>
-	/// <param name="localSvgPath">The local SVG path relative to output directory</param>
+	/// <param name="localSvgPath">The local SVG path relative to the output directory</param>
 	/// <param name="encodedUrl">The encoded Kroki URL for downloading</param>
-	/// <param name="outputDirectory">The full path to output directory</param>
-	public void RegisterDiagramForCaching(string localSvgPath, string encodedUrl, string outputDirectory)
+	/// <param name="outputDirectory">The full path to the output directory</param>
+	public void RegisterDiagramForCaching(IFileInfo outputFile, string encodedUrl)
 	{
-		if (string.IsNullOrEmpty(localSvgPath) || string.IsNullOrEmpty(encodedUrl))
+		if (string.IsNullOrEmpty(encodedUrl))
 			return;
 
-		_ = _activeDiagrams.TryAdd(localSvgPath, true);
-		_ = _diagramsToCache.TryAdd(localSvgPath, new DiagramCacheInfo(localSvgPath, encodedUrl, outputDirectory));
-	}
+		if (!outputFile.IsSubPathOf(context.DocumentationOutputDirectory))
+			return;
 
-	/// <summary>
-	/// Clear all registered diagrams (called at start of build)
-	/// </summary>
-	public void Clear()
-	{
-		_activeDiagrams.Clear();
-		_diagramsToCache.Clear();
+		_ = _diagramsToCache.TryAdd(outputFile.FullName, new DiagramCacheInfo(outputFile, encodedUrl));
 	}
 
 	/// <summary>
 	/// Create cached diagram files by downloading from Kroki in parallel
 	/// </summary>
-	/// <param name="logger">Logger for reporting download activity</param>
-	/// <param name="readFileSystem">File system for checking existing files</param>
 	/// <returns>Number of diagrams downloaded</returns>
-	public async Task<int> CreateDiagramCachedFiles(ILogger logger, IFileSystem readFileSystem)
+	public async Task<int> CreateDiagramCachedFiles(Cancel ctx)
 	{
 		if (_diagramsToCache.IsEmpty)
 			return 0;
@@ -67,87 +56,87 @@ public async Task<int> CreateDiagramCachedFiles(ILogger logger, IFileSystem read
 		await Parallel.ForEachAsync(_diagramsToCache.Values, new ParallelOptions
 		{
 			MaxDegreeOfParallelism = Environment.ProcessorCount,
-			CancellationToken = CancellationToken.None
+			CancellationToken = ctx
 		}, async (diagramInfo, ct) =>
 		{
+			var localPath = _readFileSystem.Path.GetRelativePath(context.DocumentationOutputDirectory.FullName, diagramInfo.OutputFile.FullName);
+
 			try
 			{
-				var fullPath = _writeFileSystem.Path.Combine(diagramInfo.OutputDirectory, diagramInfo.LocalSvgPath);
+				if (!diagramInfo.OutputFile.IsSubPathOf(context.DocumentationOutputDirectory))
+					return;
 
-				// Skip if file already exists
-				if (readFileSystem.File.Exists(fullPath))
+				// Skip if the file already exists
+				if (_readFileSystem.File.Exists(diagramInfo.OutputFile.FullName))
 					return;
 
-				// Create directory if needed
-				var directory = _writeFileSystem.Path.GetDirectoryName(fullPath);
+				// Create the directory if needed
+				var directory = _writeFileSystem.Path.GetDirectoryName(diagramInfo.OutputFile.FullName);
 				if (directory != null && !_writeFileSystem.Directory.Exists(directory))
-				{
 					_ = _writeFileSystem.Directory.CreateDirectory(directory);
-				}
 
 				// Download SVG content
 				var svgContent = await _httpClient.GetStringAsync(diagramInfo.EncodedUrl, ct);
 
 				// Validate SVG content
 				if (string.IsNullOrWhiteSpace(svgContent) || !svgContent.Contains("<svg", StringComparison.OrdinalIgnoreCase))
 				{
-					logger.LogWarning("Invalid SVG content received for diagram {LocalPath}", diagramInfo.LocalSvgPath);
+					_logger.LogWarning("Invalid SVG content received for diagram {LocalPath}", localPath);
 					return;
 				}
 
-				// Write atomically using temp file
-				var tempPath = fullPath + ".tmp";
+				// Write atomically using a temp file
+				var tempPath = $"{diagramInfo.OutputFile.FullName}.tmp";
 				await _writeFileSystem.File.WriteAllTextAsync(tempPath, svgContent, ct);
-				_writeFileSystem.File.Move(tempPath, fullPath);
+				_writeFileSystem.File.Move(tempPath, diagramInfo.OutputFile.FullName);
 
 				_ = Interlocked.Increment(ref downloadCount);
-				logger.LogDebug("Downloaded diagram: {LocalPath}", diagramInfo.LocalSvgPath);
+				_logger.LogDebug("Downloaded diagram: {LocalPath}", localPath);
 			}
 			catch (HttpRequestException ex)
 			{
-				logger.LogWarning("Failed to download diagram {LocalPath}: {Error}", diagramInfo.LocalSvgPath, ex.Message);
+				_logger.LogWarning("Failed to download diagram {LocalPath}: {Error}", localPath, ex.Message);
 			}
 			catch (TaskCanceledException ex) when (ex.InnerException is TimeoutException)
 			{
-				logger.LogWarning("Timeout downloading diagram {LocalPath}", diagramInfo.LocalSvgPath);
+				_logger.LogWarning("Timeout downloading diagram {LocalPath}", localPath);
 			}
 			catch (Exception ex)
 			{
-				logger.LogWarning("Unexpected error downloading diagram {LocalPath}: {Error}", diagramInfo.LocalSvgPath, ex.Message);
+				_logger.LogWarning("Unexpected error downloading diagram {LocalPath}: {Error}", localPath, ex.Message);
 			}
 		});
 
 		if (downloadCount > 0)
-		{
-			logger.LogInformation("Downloaded {DownloadCount} diagram files from Kroki", downloadCount);
-		}
+			_logger.LogInformation("Downloaded {DownloadCount} diagram files from Kroki", downloadCount);
 
 		return downloadCount;
 	}
 
 	/// <summary>
 	/// Clean up unused diagram files from the cache directory
 	/// </summary>
-	/// <param name="outputDirectory">The output directory containing cached diagrams</param>
 	/// <returns>Number of files cleaned up</returns>
-	public int CleanupUnusedDiagrams(IDirectoryInfo outputDirectory)
+	public int CleanupUnusedDiagrams()
 	{
-		var graphsDir = _writeFileSystem.Path.Combine(outputDirectory.FullName, "images", "generated-graphs");
-		if (!_writeFileSystem.Directory.Exists(graphsDir))
+		if (!_readFileSystem.Directory.Exists(context.DocumentationOutputDirectory.FullName))
+			return 0;
+		var folders = _writeFileSystem.Directory.GetDirectories(context.DocumentationOutputDirectory.FullName, "generated-graphs", SearchOption.AllDirectories);
+		var existingFiles = folders
+			.Select(f => (Folder: f, Files: _writeFileSystem.Directory.GetFiles(f, "*.svg", SearchOption.TopDirectoryOnly)))
+			.ToArray();
+		if (existingFiles.Length == 0)
 			return 0;
-
-		var existingFiles = _writeFileSystem.Directory.GetFiles(graphsDir, "*.svg", SearchOption.AllDirectories);
 		var cleanedCount = 0;
 
 		try
 		{
-			foreach (var file in existingFiles)
+			foreach (var (folder, files) in existingFiles)
 			{
-				var relativePath = _writeFileSystem.Path.GetRelativePath(outputDirectory.FullName, file);
-				var normalizedPath = relativePath.Replace(_writeFileSystem.Path.DirectorySeparatorChar, '/');
-
-				if (!_activeDiagrams.ContainsKey(normalizedPath))
+				foreach (var file in files)
 				{
+					if (_diagramsToCache.ContainsKey(file))
+						continue;
 					try
 					{
 						_writeFileSystem.File.Delete(file);
@@ -158,10 +147,9 @@ public int CleanupUnusedDiagrams(IDirectoryInfo outputDirectory)
 						// Silent failure - cleanup is opportunistic
 					}
 				}
+				// Clean up empty directories
+				CleanupEmptyDirectories(folder);
 			}
-
-			// Clean up empty directories
-			CleanupEmptyDirectories(graphsDir);
 		}
 		catch
 		{
@@ -175,22 +163,26 @@ private void CleanupEmptyDirectories(string directory)
 	{
 		try
 		{
-			foreach (var subDir in _writeFileSystem.Directory.GetDirectories(directory))
-			{
-				CleanupEmptyDirectories(subDir);
+			var folder = _writeFileSystem.DirectoryInfo.New(directory);
+			if (!folder.IsSubPathOf(context.DocumentationOutputDirectory))
+				return;
 
-				if (!_writeFileSystem.Directory.EnumerateFileSystemEntries(subDir).Any())
-				{
-					try
-					{
-						_writeFileSystem.Directory.Delete(subDir);
-					}
-					catch
-					{
-						// Silent failure - cleanup is opportunistic
-					}
-				}
-			}
+			if (folder.Name != "generated-graphs")
+				return;
+
+			if (_writeFileSystem.Directory.EnumerateFileSystemEntries(folder.FullName).Any())
+				return;
+
+			_writeFileSystem.Directory.Delete(folder.FullName);
+
+			var parentFolder = folder.Parent;
+			if (parentFolder is null || parentFolder.Name != "images")
+				return;
+
+			if (_writeFileSystem.Directory.EnumerateFileSystemEntries(parentFolder.FullName).Any())
+				return;
+
+			_writeFileSystem.Directory.Delete(folder.FullName);
 		}
 		catch
 		{

src/Elastic.Documentation/Extensions/IFileInfoExtensions.cs

@@ -18,4 +18,29 @@ public static string ReadToEnd(this IFileInfo fileInfo)
 		using var reader = new StreamReader(stream);
 		return reader.ReadToEnd();
 	}
+
+	/// Validates <paramref name="file"/> is in a subdirectory of <paramref name="parentDirectory"/>
+	public static bool IsSubPathOf(this IFileInfo file, IDirectoryInfo parentDirectory)
+	{
+		var parent = file.Directory;
+		return parent is not null && parent.IsSubPathOf(parentDirectory);
+	}
+}
+
+public static class IDirectoryInfoExtensions
+{
+	/// Validates <paramref name="directory"/> is subdirectory of <paramref name="parentDirectory"/>
+	public static bool IsSubPathOf(this IDirectoryInfo directory, IDirectoryInfo parentDirectory)
+	{
+		var parent = directory;
+		do
+		{
+			if (parent.FullName == parentDirectory.FullName)
+				return true;
+			parent = parent.Parent;
+		}
+		while (parent != null);
+
+		return false;
+	}
 }

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -108,9 +108,6 @@ public async Task ResolveDirectoryTree(Cancel ctx)
 
 	public async Task<GenerationResult> GenerateAll(Cancel ctx)
 	{
-		// Clear diagram registry for fresh tracking
-		DocumentationSet.Context.DiagramRegistry.Clear();
-
 		var result = new GenerationResult();
 
 		var generationState = Context.SkipDocumentationState ? null : GetPreviousGenerationState();
@@ -147,7 +144,7 @@ public async Task<GenerationResult> GenerateAll(Cancel ctx)
 		_logger.LogInformation($"Generating links.json");
 		var linkReference = await GenerateLinkReference(ctx);
 
-		await CreateDiagramCachedFiles();
+		await CreateDiagramCachedFiles(ctx);
 		CleanupUnusedDiagrams();
 
 		// ReSharper disable once WithExpressionModifiesAllMembers
@@ -160,9 +157,9 @@ public async Task<GenerationResult> GenerateAll(Cancel ctx)
 	/// <summary>
 	/// Downloads diagram files in parallel from Kroki
 	/// </summary>
-	public async Task CreateDiagramCachedFiles()
+	public async Task CreateDiagramCachedFiles(Cancel ctx)
 	{
-		var downloadedCount = await DocumentationSet.Context.DiagramRegistry.CreateDiagramCachedFiles(_logger, DocumentationSet.Context.ReadFileSystem);
+		var downloadedCount = await DocumentationSet.DiagramRegistry.CreateDiagramCachedFiles(ctx);
 		if (downloadedCount > 0)
 		{
 			_logger.LogInformation("Downloaded {DownloadedCount} diagram files from Kroki", downloadedCount);
@@ -174,11 +171,9 @@ public async Task CreateDiagramCachedFiles()
 	/// </summary>
 	public void CleanupUnusedDiagrams()
 	{
-		var cleanedCount = DocumentationSet.Context.DiagramRegistry.CleanupUnusedDiagrams(DocumentationSet.OutputDirectory);
+		var cleanedCount = DocumentationSet.DiagramRegistry.CleanupUnusedDiagrams();
 		if (cleanedCount > 0)
-		{
 			_logger.LogInformation("Cleaned up {CleanedCount} unused diagram files", cleanedCount);
-		}
 	}
 
 	private async Task ProcessDocumentationFiles(HashSet<string> offendingFiles, DateTimeOffset outputSeenChanges, Cancel ctx)

src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs

@@ -29,14 +29,14 @@ protected override Task<MarkdownDocument> GetMinimalParseDocumentAsync(Cancel ct
 	{
 		Title = "Prebuilt detection rules reference";
 		var markdown = GetMarkdown();
-		var document = MarkdownParser.MinimalParseStringAsync(markdown, SourceFile, null);
+		var document = MarkdownParser.MinimalParseString(markdown, SourceFile, null);
 		return Task.FromResult(document);
 	}
 
 	protected override Task<MarkdownDocument> GetParseDocumentAsync(Cancel ctx)
 	{
 		var markdown = GetMarkdown();
-		var document = MarkdownParser.ParseStringAsync(markdown, SourceFile, null);
+		var document = MarkdownParser.ParseString(markdown, SourceFile, null);
 		return Task.FromResult(document);
 	}
 
@@ -127,14 +127,14 @@ protected override Task<MarkdownDocument> GetMinimalParseDocumentAsync(Cancel ct
 	{
 		Title = Rule?.Name;
 		var markdown = GetMarkdown();
-		var document = MarkdownParser.MinimalParseStringAsync(markdown, RuleSourceMarkdownPath, null);
+		var document = MarkdownParser.MinimalParseString(markdown, RuleSourceMarkdownPath, null);
 		return Task.FromResult(document);
 	}
 
 	protected override Task<MarkdownDocument> GetParseDocumentAsync(Cancel ctx)
 	{
 		var markdown = GetMarkdown();
-		var document = MarkdownParser.ParseStringAsync(markdown, RuleSourceMarkdownPath, null);
+		var document = MarkdownParser.ParseString(markdown, RuleSourceMarkdownPath, null);
 		return Task.FromResult(document);
 	}
 

src/Elastic.Markdown/HtmlWriter.cs

@@ -43,7 +43,7 @@ public class HtmlWriter(
 	public string Render(string markdown, IFileInfo? source)
 	{
 		source ??= DocumentationSet.Context.ConfigurationPath;
-		var parsed = DocumentationSet.MarkdownParser.ParseStringAsync(markdown, source, null);
+		var parsed = DocumentationSet.MarkdownParser.ParseString(markdown, source, null);
 		return MarkdownFile.CreateHtml(parsed);
 	}