Skip to content

Commit 5eae0cd

Browse files
theletterftheletterf
committed
Tech review changes
1 parent b205ac2 commit 5eae0cd

File tree

8 files changed

+267
-283
lines changed

8 files changed

+267
-283
lines changed

‎src/Elastic.Documentation.Configuration/BuildContext.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using System.Reflection;
77
using Elastic.Documentation.Configuration.Assembler;
88
using Elastic.Documentation.Configuration.Builder;
9+
using Elastic.Documentation.Configuration.Diagram;
910
using Elastic.Documentation.Configuration.Versions;
1011
using Elastic.Documentation.Diagnostics;
1112

@@ -64,6 +65,8 @@ public string? UrlPathPrefix
6465
init => _urlPathPrefix = value;
6566
}
6667

68+
public DiagramRegistry DiagramRegistry { get; }
69+
6770
public BuildContext(IDiagnosticsCollector collector, IFileSystem fileSystem, VersionsConfiguration versionsConfig)
6871
: this(collector, fileSystem, fileSystem, versionsConfig, null, null)
6972
{
@@ -105,5 +108,6 @@ public BuildContext(
105108
{
106109
Enabled = false
107110
};
111+
DiagramRegistry = new DiagramRegistry(writeFileSystem);
108112
}
109113
}

‎src/Elastic.Documentation.Configuration/Diagram/DiagramRegistry.cs‎

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
using System.Collections.Concurrent;
6+
using System.IO.Abstractions;
7+
using Microsoft.Extensions.Logging;
8+
9+
namespace Elastic.Documentation.Configuration.Diagram;
10+
11+
/// <summary>
12+
/// Information about a diagram that needs to be cached
13+
/// </summary>
14+
/// <param name="LocalSvgPath">Local SVG path relative to output directory</param>
15+
/// <param name="EncodedUrl">Encoded Kroki URL for downloading</param>
16+
/// <param name="OutputDirectory">Full path to output directory</param>
17+
public record DiagramCacheInfo(string LocalSvgPath, string EncodedUrl, string OutputDirectory);
18+
19+
/// <summary>
20+
/// Registry to track active diagrams and manage cleanup of outdated cached files
21+
/// </summary>
22+
/// <param name="writeFileSystem">File system for write/delete operations during cleanup</param>
23+
public class DiagramRegistry(IFileSystem writeFileSystem) : IDisposable
24+
{
25+
private readonly ConcurrentDictionary<string, bool> _activeDiagrams = new();
26+
private readonly ConcurrentDictionary<string, DiagramCacheInfo> _diagramsToCache = new();
27+
private readonly IFileSystem _writeFileSystem = writeFileSystem;
28+
private readonly HttpClient _httpClient = new() { Timeout = TimeSpan.FromSeconds(30) };
29+
30+
/// <summary>
31+
/// Register a diagram for caching (collects info for later batch processing)
32+
/// </summary>
33+
/// <param name="localSvgPath">The local SVG path relative to output directory</param>
34+
/// <param name="encodedUrl">The encoded Kroki URL for downloading</param>
35+
/// <param name="outputDirectory">The full path to output directory</param>
36+
public void RegisterDiagramForCaching(string localSvgPath, string encodedUrl, string outputDirectory)
37+
{
38+
if (string.IsNullOrEmpty(localSvgPath) || string.IsNullOrEmpty(encodedUrl))
39+
return;
40+
41+
_ = _activeDiagrams.TryAdd(localSvgPath, true);
42+
_ = _diagramsToCache.TryAdd(localSvgPath, new DiagramCacheInfo(localSvgPath, encodedUrl, outputDirectory));
43+
}
44+
45+
/// <summary>
46+
/// Clear all registered diagrams (called at start of build)
47+
/// </summary>
48+
public void Clear()
49+
{
50+
_activeDiagrams.Clear();
51+
_diagramsToCache.Clear();
52+
}
53+
54+
/// <summary>
55+
/// Create cached diagram files by downloading from Kroki in parallel
56+
/// </summary>
57+
/// <param name="logger">Logger for reporting download activity</param>
58+
/// <param name="readFileSystem">File system for checking existing files</param>
59+
/// <returns>Number of diagrams downloaded</returns>
60+
public async Task<int> CreateDiagramCachedFiles(ILogger logger, IFileSystem readFileSystem)
61+
{
62+
if (_diagramsToCache.IsEmpty)
63+
return 0;
64+
65+
var downloadCount = 0;
66+
67+
await Parallel.ForEachAsync(_diagramsToCache.Values, new ParallelOptions
68+
{
69+
MaxDegreeOfParallelism = Environment.ProcessorCount,
70+
CancellationToken = CancellationToken.None
71+
}, async (diagramInfo, ct) =>
72+
{
73+
try
74+
{
75+
var fullPath = _writeFileSystem.Path.Combine(diagramInfo.OutputDirectory, diagramInfo.LocalSvgPath);
76+
77+
// Skip if file already exists
78+
if (readFileSystem.File.Exists(fullPath))
79+
return;
80+
81+
// Create directory if needed
82+
var directory = _writeFileSystem.Path.GetDirectoryName(fullPath);
83+
if (directory != null && !_writeFileSystem.Directory.Exists(directory))
84+
{
85+
_ = _writeFileSystem.Directory.CreateDirectory(directory);
86+
}
87+
88+
// Download SVG content
89+
var svgContent = await _httpClient.GetStringAsync(diagramInfo.EncodedUrl, ct);
90+
91+
// Validate SVG content
92+
if (string.IsNullOrWhiteSpace(svgContent) || !svgContent.Contains("<svg", StringComparison.OrdinalIgnoreCase))
93+
{
94+
logger.LogWarning("Invalid SVG content received for diagram {LocalPath}", diagramInfo.LocalSvgPath);
95+
return;
96+
}
97+
98+
// Write atomically using temp file
99+
var tempPath = fullPath + ".tmp";
100+
await _writeFileSystem.File.WriteAllTextAsync(tempPath, svgContent, ct);
101+
_writeFileSystem.File.Move(tempPath, fullPath);
102+
103+
_ = Interlocked.Increment(ref downloadCount);
104+
logger.LogDebug("Downloaded diagram: {LocalPath}", diagramInfo.LocalSvgPath);
105+
}
106+
catch (HttpRequestException ex)
107+
{
108+
logger.LogWarning("Failed to download diagram {LocalPath}: {Error}", diagramInfo.LocalSvgPath, ex.Message);
109+
}
110+
catch (TaskCanceledException ex) when (ex.InnerException is TimeoutException)
111+
{
112+
logger.LogWarning("Timeout downloading diagram {LocalPath}", diagramInfo.LocalSvgPath);
113+
}
114+
catch (Exception ex)
115+
{
116+
logger.LogWarning("Unexpected error downloading diagram {LocalPath}: {Error}", diagramInfo.LocalSvgPath, ex.Message);
117+
}
118+
});
119+
120+
if (downloadCount > 0)
121+
{
122+
logger.LogInformation("Downloaded {DownloadCount} diagram files from Kroki", downloadCount);
123+
}
124+
125+
return downloadCount;
126+
}
127+
128+
/// <summary>
129+
/// Clean up unused diagram files from the cache directory
130+
/// </summary>
131+
/// <param name="outputDirectory">The output directory containing cached diagrams</param>
132+
/// <returns>Number of files cleaned up</returns>
133+
public int CleanupUnusedDiagrams(IDirectoryInfo outputDirectory)
134+
{
135+
var graphsDir = _writeFileSystem.Path.Combine(outputDirectory.FullName, "images", "generated-graphs");
136+
if (!_writeFileSystem.Directory.Exists(graphsDir))
137+
return 0;
138+
139+
var existingFiles = _writeFileSystem.Directory.GetFiles(graphsDir, "*.svg", SearchOption.AllDirectories);
140+
var cleanedCount = 0;
141+
142+
try
143+
{
144+
foreach (var file in existingFiles)
145+
{
146+
var relativePath = _writeFileSystem.Path.GetRelativePath(outputDirectory.FullName, file);
147+
var normalizedPath = relativePath.Replace(_writeFileSystem.Path.DirectorySeparatorChar, '/');
148+
149+
if (!_activeDiagrams.ContainsKey(normalizedPath))
150+
{
151+
try
152+
{
153+
_writeFileSystem.File.Delete(file);
154+
cleanedCount++;
155+
}
156+
catch
157+
{
158+
// Silent failure - cleanup is opportunistic
159+
}
160+
}
161+
}
162+
163+
// Clean up empty directories
164+
CleanupEmptyDirectories(graphsDir);
165+
}
166+
catch
167+
{
168+
// Silent failure - cleanup is opportunistic
169+
}
170+
171+
return cleanedCount;
172+
}
173+
174+
private void CleanupEmptyDirectories(string directory)
175+
{
176+
try
177+
{
178+
foreach (var subDir in _writeFileSystem.Directory.GetDirectories(directory))
179+
{
180+
CleanupEmptyDirectories(subDir);
181+
182+
if (!_writeFileSystem.Directory.EnumerateFileSystemEntries(subDir).Any())
183+
{
184+
try
185+
{
186+
_writeFileSystem.Directory.Delete(subDir);
187+
}
188+
catch
189+
{
190+
// Silent failure - cleanup is opportunistic
191+
}
192+
}
193+
}
194+
}
195+
catch
196+
{
197+
// Silent failure - cleanup is opportunistic
198+
}
199+
}
200+
201+
/// <summary>
202+
/// Dispose of resources, including the HttpClient
203+
/// </summary>
204+
public void Dispose()
205+
{
206+
_httpClient.Dispose();
207+
GC.SuppressFinalize(this);
208+
}
209+
}

‎src/Elastic.Markdown/DocumentationGenerator.cs‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using System.Text.Json;
77
using Elastic.Documentation;
88
using Elastic.Documentation.Configuration;
9+
using Elastic.Documentation.Configuration.Diagram;
910
using Elastic.Documentation.Legacy;
1011
using Elastic.Documentation.Links;
1112
using Elastic.Documentation.Serialization;
@@ -108,7 +109,7 @@ public async Task ResolveDirectoryTree(Cancel ctx)
108109
public async Task<GenerationResult> GenerateAll(Cancel ctx)
109110
{
110111
// Clear diagram registry for fresh tracking
111-
DiagramRegistry.Clear();
112+
DocumentationSet.Context.DiagramRegistry.Clear();
112113

113114
var result = new GenerationResult();
114115

@@ -146,8 +147,15 @@ public async Task<GenerationResult> GenerateAll(Cancel ctx)
146147
_logger.LogInformation($"Generating links.json");
147148
var linkReference = await GenerateLinkReference(ctx);
148149

150+
// Download diagram files in parallel
151+
var downloadedCount = await DocumentationSet.Context.DiagramRegistry.CreateDiagramCachedFiles(_logger, DocumentationSet.Context.ReadFileSystem);
152+
if (downloadedCount > 0)
153+
{
154+
_logger.LogInformation("Downloaded {DownloadedCount} diagram files from Kroki", downloadedCount);
155+
}
156+
149157
// Clean up unused diagram files
150-
var cleanedCount = DiagramRegistry.CleanupUnusedDiagrams(DocumentationSet.OutputDirectory.FullName);
158+
var cleanedCount = DocumentationSet.Context.DiagramRegistry.CleanupUnusedDiagrams(DocumentationSet.OutputDirectory);
151159
if (cleanedCount > 0)
152160
{
153161
_logger.LogInformation("Cleaned up {CleanedCount} unused diagram files", cleanedCount);

‎src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs‎

Lines changed: 6 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
using System.Security.Cryptography;
66
using System.Text;
7+
using Elastic.Documentation.Configuration.Diagram;
78
using Elastic.Markdown.Diagnostics;
89

910
namespace Elastic.Markdown.Myst.Directives.Diagram;
@@ -68,12 +69,9 @@ public override void FinalizeAndValidate(ParserContext context)
6869
return;
6970
}
7071

71-
// Register diagram for tracking and cleanup
72-
DiagramRegistry.RegisterDiagram(LocalSvgPath);
73-
74-
// Cache diagram asynchronously - fire and forget
75-
// Use simplified approach without lock files to avoid orphaned locks
76-
_ = Task.Run(() => TryCacheDiagramAsync(context));
72+
// Register diagram for tracking, cleanup, and batch caching
73+
var outputDirectory = context.Build.DocumentationOutputDirectory.FullName;
74+
context.DiagramRegistry.RegisterDiagramForCaching(LocalSvgPath, EncodedUrl, outputDirectory);
7775
}
7876

7977
private string? ExtractContent()
@@ -106,9 +104,9 @@ private string GenerateContentHash(string diagramType, string content)
106104
private string GenerateLocalPath(ParserContext context)
107105
{
108106
var markdownFileName = "unknown";
109-
if (context.MarkdownSourcePath?.FullName != null)
107+
if (context.MarkdownSourcePath?.Name is not null)
110108
{
111-
markdownFileName = Path.GetFileNameWithoutExtension(context.MarkdownSourcePath.FullName);
109+
markdownFileName = Path.GetFileNameWithoutExtension(context.MarkdownSourcePath.Name);
112110
}
113111

114112
var filename = $"{markdownFileName}-diagram-{DiagramType}-{ContentHash}.svg";
@@ -118,63 +116,5 @@ private string GenerateLocalPath(ParserContext context)
118116
return localPath.Replace(Path.DirectorySeparatorChar, '/');
119117
}
120118

121-
private async Task TryCacheDiagramAsync(ParserContext context)
122-
{
123-
if (string.IsNullOrEmpty(EncodedUrl) || string.IsNullOrEmpty(LocalSvgPath))
124-
return;
125-
126-
try
127-
{
128-
// Determine the full output path
129-
var outputDirectory = context.Build.DocumentationOutputDirectory.FullName;
130-
var fullPath = Path.Combine(outputDirectory, LocalSvgPath);
131-
132-
// Skip if file already exists - simple check without locking
133-
if (File.Exists(fullPath))
134-
return;
135-
136-
// Create directory if it doesn't exist
137-
var directory = Path.GetDirectoryName(fullPath);
138-
if (directory != null && !Directory.Exists(directory))
139-
{
140-
_ = Directory.CreateDirectory(directory);
141-
}
142-
143-
// Download SVG from Kroki using shared HttpClient
144-
var svgContent = await DiagramHttpClient.Instance.GetStringAsync(EncodedUrl);
145-
146-
// Basic validation - ensure we got SVG content
147-
// SVG can start with XML declaration, DOCTYPE, or directly with <svg>
148-
if (string.IsNullOrWhiteSpace(svgContent) || !svgContent.Contains("<svg", StringComparison.OrdinalIgnoreCase))
149-
{
150-
// Invalid content - don't cache
151-
return;
152-
}
153119

154-
// Write to local file atomically using a temp file
155-
var tempPath = fullPath + ".tmp";
156-
await File.WriteAllTextAsync(tempPath, svgContent);
157-
File.Move(tempPath, fullPath);
158-
}
159-
catch (HttpRequestException)
160-
{
161-
// Network-related failures - silent fallback to Kroki URLs
162-
// Caching is opportunistic, network issues shouldn't generate warnings
163-
}
164-
catch (TaskCanceledException ex) when (ex.InnerException is TimeoutException)
165-
{
166-
// Timeout - silent fallback to Kroki URLs
167-
// Timeouts are expected in slow network conditions
168-
}
169-
catch (IOException)
170-
{
171-
// File system issues - silent fallback to Kroki URLs
172-
// Disk space or permission issues shouldn't break builds
173-
}
174-
catch (Exception)
175-
{
176-
// Unexpected errors - silent fallback to Kroki URLs
177-
// Caching is opportunistic, any failure should fallback gracefully
178-
}
179-
}
180120
}

‎src/Elastic.Markdown/Myst/Directives/Diagram/DiagramHttpClient.cs‎

Lines changed: 0 additions & 21 deletions
This file was deleted.

0 commit comments

Comments
 (0)