Skip to content

Commit 034d552

Browse files
MpdreamzMpdreamz
committed
Track git information as part of incremental builds and link references
1 parent 7ddb744 commit 034d552

File tree

9 files changed

+206
-46
lines changed

9 files changed

+206
-46
lines changed

‎src/Elastic.Markdown/BuildContext.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ public record BuildContext
1717

1818
public IFileInfo ConfigurationPath { get; }
1919

20+
public GitConfiguration Git { get; }
21+
2022
public required DiagnosticsCollector Collector { get; init; }
2123

2224
public bool Force { get; init; }
@@ -54,6 +56,8 @@ public BuildContext(IFileSystem readFileSystem, IFileSystem writeFileSystem, str
5456
if (ConfigurationPath.FullName != SourcePath.FullName)
5557
SourcePath = ConfigurationPath.Directory!;
5658

59+
Git = GitConfiguration.Create(ReadFileSystem);
60+
5761

5862
}
5963

‎src/Elastic.Markdown/DocumentationGenerator.cs‎

Lines changed: 65 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
33
// See the LICENSE file in the project root for more information
44
using System.IO.Abstractions;
5-
using System.Security.Cryptography;
65
using System.Text.Json;
76
using System.Text.Json.Serialization;
87
using Elastic.Markdown.IO;
@@ -12,13 +11,20 @@
1211
namespace Elastic.Markdown;
1312

1413
[JsonSourceGenerationOptions(WriteIndented = true)]
15-
[JsonSerializable(typeof(OutputState))]
14+
[JsonSerializable(typeof(GenerationState))]
15+
[JsonSerializable(typeof(LinkReference))]
16+
[JsonSerializable(typeof(GitConfiguration))]
1617
internal partial class SourceGenerationContext : JsonSerializerContext;
1718

18-
public class OutputState
19+
public record GenerationState
1920
{
20-
public DateTimeOffset LastSeenChanges { get; set; }
21-
public string[] Conflict { get; set; } = [];
21+
[JsonPropertyName("last_seen_changes")]
22+
public required DateTimeOffset LastSeenChanges { get; init; }
23+
[JsonPropertyName("invalid_files")]
24+
public required string[] InvalidFiles { get; init; } = [];
25+
26+
[JsonPropertyName("git")]
27+
public required GitConfiguration Git { get; init; }
2228
}
2329

2430
public class DocumentationGenerator
@@ -49,18 +55,13 @@ ILoggerFactory logger
4955
_logger.LogInformation($"Output directory: {docSet.OutputPath} Exists: {docSet.OutputPath.Exists}");
5056
}
5157

52-
public OutputState? OutputState
58+
public GenerationState? GetPreviousGenerationState()
5359
{
54-
get
55-
{
56-
var stateFile = DocumentationSet.OutputStateFile;
57-
stateFile.Refresh();
58-
if (!stateFile.Exists) return null;
59-
var contents = stateFile.FileSystem.File.ReadAllText(stateFile.FullName);
60-
return JsonSerializer.Deserialize(contents, SourceGenerationContext.Default.OutputState);
61-
62-
63-
}
60+
var stateFile = DocumentationSet.OutputStateFile;
61+
stateFile.Refresh();
62+
if (!stateFile.Exists) return null;
63+
var contents = stateFile.FileSystem.File.ReadAllText(stateFile.FullName);
64+
return JsonSerializer.Deserialize(contents, SourceGenerationContext.Default.GenerationState);
6465
}
6566

6667

@@ -69,26 +70,12 @@ public async Task ResolveDirectoryTree(Cancel ctx) =>
6970

7071
public async Task GenerateAll(Cancel ctx)
7172
{
72-
if (Context.Force || OutputState == null)
73+
var generationState = GetPreviousGenerationState();
74+
if (Context.Force || generationState == null)
7375
DocumentationSet.ClearOutputDirectory();
7476

75-
_logger.LogInformation($"Last write source: {DocumentationSet.LastWrite}, output observed: {OutputState?.LastSeenChanges}");
76-
77-
var offendingFiles = new HashSet<string>(OutputState?.Conflict ?? []);
78-
var outputSeenChanges = OutputState?.LastSeenChanges ?? DateTimeOffset.MinValue;
79-
if (offendingFiles.Count > 0)
80-
{
81-
_logger.LogInformation($"Reapplying changes since {DocumentationSet.LastWrite}");
82-
_logger.LogInformation($"Reapplying for {offendingFiles.Count} files with errors/warnings");
83-
}
84-
else if (DocumentationSet.LastWrite > outputSeenChanges && OutputState != null)
85-
_logger.LogInformation($"Using incremental build picking up changes since: {OutputState.LastSeenChanges}");
86-
else if (DocumentationSet.LastWrite <= outputSeenChanges && OutputState != null)
87-
{
88-
_logger.LogInformation($"No changes in source since last observed write {OutputState.LastSeenChanges} "
89-
+ "Pass --force to force a full regeneration");
77+
if (CompilationNotNeeded(generationState, out var offendingFiles, out var outputSeenChanges))
9078
return;
91-
}
9279

9380
_logger.LogInformation("Resolving tree");
9481
await ResolveDirectoryTree(ctx);
@@ -122,6 +109,7 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) =>
122109
Context.Collector.Channel.TryComplete();
123110

124111
await GenerateDocumentationState(ctx);
112+
await GenerateLinkReference(ctx);
125113

126114
await Context.Collector.StopAsync(ctx);
127115

@@ -133,18 +121,58 @@ IFileInfo OutputFile(string relativePath)
133121

134122
}
135123

124+
private bool CompilationNotNeeded(GenerationState? generationState, out HashSet<string> offendingFiles,
125+
out DateTimeOffset outputSeenChanges)
126+
{
127+
offendingFiles = new HashSet<string>(generationState?.InvalidFiles ?? []);
128+
outputSeenChanges = generationState?.LastSeenChanges ?? DateTimeOffset.MinValue;
129+
if (generationState == null)
130+
return false;
131+
132+
if (Context.Git != generationState.Git)
133+
{
134+
_logger.LogInformation($"Full compilation: current git context: {Context.Git} differs from previous git context: {generationState.Git}");
135+
return false;
136+
}
137+
138+
if (offendingFiles.Count > 0)
139+
{
140+
_logger.LogInformation($"Incremental compilation. since: {DocumentationSet.LastWrite}");
141+
_logger.LogInformation($"Incremental compilation. {offendingFiles.Count} files with errors/warnings");
142+
}
143+
else if (DocumentationSet.LastWrite > outputSeenChanges)
144+
_logger.LogInformation($"Incremental compilation. since: {generationState.LastSeenChanges}");
145+
else if (DocumentationSet.LastWrite <= outputSeenChanges)
146+
{
147+
_logger.LogInformation($"No compilation: no changes since last observed: {generationState.LastSeenChanges}");
148+
_logger.LogInformation($"No compilation: no changes since last observed: {generationState.LastSeenChanges} "
149+
+ "Pass --force to force a full regeneration");
150+
return true;
151+
}
152+
153+
return false;
154+
}
155+
156+
private async Task GenerateLinkReference(Cancel ctx)
157+
{
158+
var file = DocumentationSet.LinkReferenceFile;
159+
var state = LinkReference.Create(DocumentationSet);
160+
var bytes = JsonSerializer.SerializeToUtf8Bytes(state, SourceGenerationContext.Default.LinkReference);
161+
await DocumentationSet.OutputPath.FileSystem.File.WriteAllBytesAsync(file.FullName, bytes, ctx);
162+
}
163+
136164
private async Task GenerateDocumentationState(Cancel ctx)
137165
{
138166
var stateFile = DocumentationSet.OutputStateFile;
139167
_logger.LogInformation($"Writing documentation state {DocumentationSet.LastWrite} to {stateFile.FullName}");
140168
var badFiles = Context.Collector.OffendingFiles.ToArray();
141-
var state = new OutputState
169+
var state = new GenerationState
142170
{
143171
LastSeenChanges = DocumentationSet.LastWrite,
144-
Conflict = badFiles
145-
172+
InvalidFiles = badFiles,
173+
Git = Context.Git
146174
};
147-
var bytes = JsonSerializer.SerializeToUtf8Bytes(state, SourceGenerationContext.Default.OutputState);
175+
var bytes = JsonSerializer.SerializeToUtf8Bytes(state, SourceGenerationContext.Default.GenerationState);
148176
await DocumentationSet.OutputPath.FileSystem.File.WriteAllBytesAsync(stateFile.FullName, bytes, ctx);
149177
}
150178

‎src/Elastic.Markdown/Elastic.Markdown.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
<ItemGroup>
1818
<PackageReference Include="DotNet.Glob" Version="3.1.3" />
19+
<PackageReference Include="ini-parser-netstandard" Version="2.5.2" />
1920
<PackageReference Include="Markdig" Version="0.37.0"/>
2021
<PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.0" />
2122
<PackageReference Include="RazorSlices" Version="0.8.1" />

‎src/Elastic.Markdown/IO/DocumentationSet.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ public class DocumentationSet
1313
public BuildContext Context { get; }
1414
public string Name { get; }
1515
public IFileInfo OutputStateFile { get; }
16+
public IFileInfo LinkReferenceFile { get; }
1617

1718
public IDirectoryInfo SourcePath { get; }
1819
public IDirectoryInfo OutputPath { get; }
@@ -34,6 +35,7 @@ public DocumentationSet(BuildContext context)
3435

3536
Name = SourcePath.FullName;
3637
OutputStateFile = OutputPath.FileSystem.FileInfo.New(Path.Combine(OutputPath.FullName, ".doc.state"));
38+
LinkReferenceFile = OutputPath.FileSystem.FileInfo.New(Path.Combine(OutputPath.FullName, "links.json"));
3739

3840
Files = context.ReadFileSystem.Directory
3941
.EnumerateFiles(SourcePath.FullName, "*.*", SearchOption.AllDirectories)

‎src/Elastic.Markdown/IO/GitConfiguration.cs‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
using System.IO.Abstractions;
6+
using System.Text.Json.Serialization;
7+
using IniParser;
8+
9+
namespace Elastic.Markdown.IO;
10+
11+
public record GitConfiguration
12+
{
13+
[JsonPropertyName("branch")]
14+
public required string Branch { get; init; }
15+
[JsonPropertyName("remote")]
16+
public required string Remote { get; init; }
17+
[JsonPropertyName("ref")]
18+
public required string Ref { get; init; }
19+
20+
// manual read because libgit2sharp is not yet AOT ready
21+
public static GitConfiguration Create(IFileSystem fileSystem)
22+
{
23+
// filesystem is not real so return a dummy
24+
if (fileSystem is not FileSystem)
25+
{
26+
var fakeRef = Guid.NewGuid().ToString().Substring(0, 16);
27+
return new GitConfiguration
28+
{
29+
Branch = $"test-{fakeRef}",
30+
Remote = "elastic/docs-builder",
31+
Ref = fakeRef,
32+
};
33+
}
34+
35+
var gitConfig = Git(".git/config");
36+
if (!gitConfig.Exists)
37+
throw new Exception($"{Paths.Root.FullName} is not a git repository.");
38+
39+
var head = Read(".git/HEAD").Replace("ref: ", string.Empty);
40+
var gitRef = Read(".git/" + head);
41+
var branch = head.Replace("refs/heads/", string.Empty);
42+
43+
var ini = new FileIniDataParser();
44+
using var stream = gitConfig.OpenRead();
45+
using var streamReader = new StreamReader(stream);
46+
var config = ini.ReadData(streamReader);
47+
var remoteName = config[$"branch \"{branch}\""]["remote"];
48+
var remote = config[$"remote \"{remoteName}\""]["url"];
49+
50+
return new GitConfiguration
51+
{
52+
Ref = gitRef,
53+
Branch = branch,
54+
Remote = remote
55+
};
56+
57+
IFileInfo Git(string path) => fileSystem.FileInfo.New(Path.Combine(Paths.Root.FullName, path));
58+
59+
string Read(string path) =>
60+
fileSystem.File.ReadAllText(Git(path).FullName).Trim(Environment.NewLine.ToCharArray());
61+
}
62+
}

‎src/Elastic.Markdown/IO/LinkReference.cs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
using System.IO.Abstractions;
6+
using System.Text.Json.Serialization;
7+
using IniParser;
8+
9+
namespace Elastic.Markdown.IO;
10+
11+
public record LinkReference
12+
{
13+
[JsonPropertyName("origin")]
14+
public required GitConfiguration Origin { get; init; }
15+
[JsonPropertyName("links")]
16+
public required string[] Links { get; init; } = [];
17+
18+
public static LinkReference Create(DocumentationSet set)
19+
{
20+
var links = set.FlatMappedFiles.Values
21+
.OfType<MarkdownFile>()
22+
.Select(m => m.RelativePath).ToArray();
23+
return new LinkReference { Origin = set.Context.Git, Links = links };
24+
}
25+
}

‎src/Elastic.Markdown/IO/Paths.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ public static class Paths
88
private static DirectoryInfo RootDirectoryInfo()
99
{
1010
var directory = new DirectoryInfo(Directory.GetCurrentDirectory());
11-
while (directory != null && !directory.GetFiles("*.sln").Any())
11+
while (directory != null &&
12+
(directory.GetFiles("*.sln").Length == 0 || directory.GetDirectories(".git").Length == 0))
1213
directory = directory.Parent;
1314
return directory ?? new DirectoryInfo(Directory.GetCurrentDirectory());
1415
}

‎tests/Elastic.Markdown.Tests/SiteMap/LinkReferenceTests.cs‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
using Elastic.Markdown.IO;
6+
using FluentAssertions;
7+
using Xunit.Abstractions;
8+
9+
namespace Elastic.Markdown.Tests.SiteMap;
10+
11+
public class LinkReferenceTests(ITestOutputHelper output) : NavigationTestsBase(output)
12+
{
13+
[Fact]
14+
public void Create()
15+
{
16+
var reference = LinkReference.Create(Set);
17+
18+
reference.Should().NotBeNull();
19+
}
20+
}
21+
22+
public class GitConfigurationTests(ITestOutputHelper output) : NavigationTestsBase(output)
23+
{
24+
[Fact]
25+
public void Create()
26+
{
27+
var git = GitConfiguration.Create(ReadFileSystem);
28+
29+
git.Should().NotBeNull();
30+
git!.Branch.Should().NotBeNullOrWhiteSpace();
31+
// this validates we are not returning the test instance as were doing a real read
32+
git.Branch.Should().NotContain(git.Ref);
33+
git.Ref.Should().NotBeNullOrWhiteSpace();
34+
git.Remote.Should().NotBeNullOrWhiteSpace();
35+
}
36+
}

‎tests/Elastic.Markdown.Tests/SiteMap/NavigationTestsBase.cs‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,28 +16,29 @@ public class NavigationTestsBase : IAsyncLifetime
1616
protected NavigationTestsBase(ITestOutputHelper output)
1717
{
1818
var logger = new TestLoggerFactory(output);
19-
var readFs = new FileSystem(); //use real IO to read docs.
19+
ReadFileSystem = new FileSystem(); //use real IO to read docs.
2020
var writeFs = new MockFileSystem(new MockFileSystemOptions //use in memory mock fs to test generation
2121
{
2222
CurrentDirectory = Paths.Root.FullName
2323
});
24-
var context = new BuildContext(readFs, writeFs)
24+
var context = new BuildContext(ReadFileSystem, writeFs)
2525
{
2626
Force = false,
2727
UrlPathPrefix = null,
2828
Collector = new DiagnosticsCollector(logger, [])
2929
};
3030

31-
var set = new DocumentationSet(context);
31+
Set = new DocumentationSet(context);
3232

33-
set.Files.Should().HaveCountGreaterThan(10);
34-
Generator = new DocumentationGenerator(set, logger);
33+
Set.Files.Should().HaveCountGreaterThan(10);
34+
Generator = new DocumentationGenerator(Set, logger);
3535

3636
}
3737

38-
public DocumentationGenerator Generator { get; }
39-
40-
public ConfigurationFile Configuration { get; set; } = default!;
38+
protected FileSystem ReadFileSystem { get; set; }
39+
protected DocumentationSet Set { get; }
40+
protected DocumentationGenerator Generator { get; }
41+
protected ConfigurationFile Configuration { get; set; } = default!;
4142

4243
public async Task InitializeAsync()
4344
{

0 commit comments

Comments
 (0)