Skip to content

Commit abb4920

Browse files
MpdreamzMpdreamz
committed
Add ability to set path for repositories in assembler.yml
Only works locally. If you specify `--skip-private-repositories` locally and we detect a solution file we automatically inject docs-builder in to the assembler navigation. This allows our integration tests to tests new features in the assembler.
1 parent 60c5246 commit abb4920

File tree

9 files changed

+143
-29
lines changed

9 files changed

+143
-29
lines changed

‎src/Elastic.Documentation.Configuration/Assembler/AssemblyConfiguration.cs‎

Lines changed: 55 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,23 @@ public static AssemblyConfiguration Deserialize(string yaml, bool skipPrivateRep
3030
var repository = RepositoryDefaults(r, name);
3131
config.ReferenceRepositories[name] = repository;
3232
}
33+
34+
// if we are not running in CI, and we are skipping private repositories, and we can locate the solution directory. build the local docs-content repository
35+
if (string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("CI"))
36+
&& skipPrivateRepositories
37+
&& config.ReferenceRepositories.TryGetValue("docs-builder", out var docsContentRepository)
38+
&& Paths.GetSolutionDirectory() is { } solutionDir
39+
)
40+
{
41+
var docsRepositoryPath = Path.Combine(solutionDir.FullName, "docs");
42+
config.ReferenceRepositories["docs-builder"] = docsContentRepository with
43+
{
44+
Skip = false,
45+
Path = docsRepositoryPath
46+
};
47+
}
48+
49+
3350
var privateRepositories = config.ReferenceRepositories.Where(r => r.Value.Private).ToList();
3451
foreach (var (name, _) in privateRepositories)
3552
{
@@ -45,7 +62,9 @@ public static AssemblyConfiguration Deserialize(string yaml, bool skipPrivateRep
4562
.Where(r => !r.Skip)
4663
.Concat([config.Narrative]).ToDictionary(kvp => kvp.Name, kvp => kvp);
4764

48-
config.PrivateRepositories = privateRepositories.ToDictionary(kvp => kvp.Key, kvp => kvp.Value);
65+
config.PrivateRepositories = privateRepositories
66+
.Where(r => !r.Value.Skip)
67+
.ToDictionary(kvp => kvp.Key, kvp => kvp.Value);
4968
return config;
5069
}
5170
catch (Exception e)
@@ -69,6 +88,12 @@ private static TRepository RepositoryDefaults<TRepository>(TRepository r, string
6988
GitReferenceNext = string.IsNullOrEmpty(repository.GitReferenceNext) ? "main" : repository.GitReferenceNext,
7089
GitReferenceEdge = string.IsNullOrEmpty(repository.GitReferenceEdge) ? "main" : repository.GitReferenceEdge,
7190
};
91+
// ensure we always null path if we are running in CI
92+
if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("CI")))
93+
repository = repository with
94+
{
95+
Path = null
96+
};
7297
if (string.IsNullOrEmpty(repository.Origin))
7398
{
7499
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("GITHUB_ACTIONS")))
@@ -124,21 +149,34 @@ public ContentSourceMatch Match(string repository, string branchOrTag)
124149
var next = r.GetBranch(ContentSource.Next);
125150
var isVersionBranch = ContentSourceRegex.MatchVersionBranch().IsMatch(branchOrTag);
126151
if (current == branchOrTag)
127-
match = match with { Current = ContentSource.Current };
152+
match = match with
153+
{
154+
Current = ContentSource.Current
155+
};
128156
if (next == branchOrTag)
129-
match = match with { Next = ContentSource.Next };
157+
match = match with
158+
{
159+
Next = ContentSource.Next
160+
};
130161
if (isVersionBranch && SemVersion.TryParse(branchOrTag + ".0", out var v))
131162
{
132163
// if the current branch is a version, only speculatively match if branch is actually a new version
133164
if (SemVersion.TryParse(current + ".0", out var currentVersion))
134165
{
135166
if (v >= currentVersion)
136-
match = match with { Speculative = true };
167+
match = match with
168+
{
169+
Speculative = true
170+
};
137171
}
138172
// assume we are newly onboarding the repository to current/next
139173
else
140-
match = match with { Speculative = true };
174+
match = match with
175+
{
176+
Speculative = true
177+
};
141178
}
179+
142180
return match;
143181
}
144182

@@ -147,19 +185,27 @@ public ContentSourceMatch Match(string repository, string branchOrTag)
147185
// this is an unknown new elastic repository
148186
var isVersionBranch = ContentSourceRegex.MatchVersionBranch().IsMatch(branchOrTag);
149187
if (isVersionBranch || branchOrTag == "main" || branchOrTag == "master")
150-
return match with { Speculative = true };
188+
return match with
189+
{
190+
Speculative = true
191+
};
151192
}
152193

153194
if (Narrative.GetBranch(ContentSource.Current) == branchOrTag)
154-
match = match with { Current = ContentSource.Current };
195+
match = match with
196+
{
197+
Current = ContentSource.Current
198+
};
155199
if (Narrative.GetBranch(ContentSource.Next) == branchOrTag)
156-
match = match with { Next = ContentSource.Next };
200+
match = match with
201+
{
202+
Next = ContentSource.Next
203+
};
157204

158205
return match;
159206
}
160207

161208
public record ContentSourceMatch(ContentSource? Current, ContentSource? Next, bool Speculative);
162-
163209
}
164210

165211
internal static partial class ContentSourceRegex

‎src/Elastic.Documentation.Configuration/Assembler/Repository.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,10 @@ public record Repository
3636
[YamlMember(Alias = "skip")]
3737
public bool Skip { get; set; }
3838

39+
/// <summary> Allows you to override the path to the repository, but only during local builds. </summary>
40+
[YamlMember(Alias = "path")]
41+
public string? Path { get; set; }
42+
3943
[YamlMember(Alias = "current")]
4044
public string GitReferenceCurrent { get; set; } = "main";
4145

‎src/Elastic.Documentation.Configuration/ConfigurationFileProvider.cs‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,9 @@ public ConfigurationFileProvider(IFileSystem fileSystem, bool skipPrivateReposit
5555

5656
public IFileInfo LegacyUrlMappingsFile { get; }
5757

58-
public IFileInfo CreateNavigationFile(IReadOnlyDictionary<string, Repository> privateRepositories)
58+
public IFileInfo CreateNavigationFile(AssemblyConfiguration configuration)
5959
{
60+
var privateRepositories = configuration.PrivateRepositories;
6061
if (privateRepositories.Count == 0 || !SkipPrivateRepositories)
6162
return NavigationFile;
6263

@@ -104,6 +105,23 @@ public IFileInfo CreateNavigationFile(IReadOnlyDictionary<string, Repository> pr
104105
if (spacing == -1 || reindenting > 0)
105106
_fileSystem.File.AppendAllLines(tempFile, [line]);
106107
}
108+
109+
if (configuration.AvailableRepositories.TryGetValue("docs-builder", out var docsBuildRepository) && docsBuildRepository is { Skip: false, Path: not null })
110+
{
111+
// language=yaml
112+
_fileSystem.File.AppendAllText(tempFile,
113+
"""
114+
- toc: docs-builder://
115+
path_prefix: reference/docs-builder
116+
children:
117+
- toc: docs-builder://development
118+
path_prefix: reference/docs-builder/dev
119+
children:
120+
- toc: docs-builder://development/link-validation
121+
path_prefix: reference/docs-builder/dev/link-val
122+
123+
""");
124+
}
107125
NavigationFile = _fileSystem.FileInfo.New(tempFile);
108126
return NavigationFile;
109127

‎src/Elastic.Documentation/GlobalCommandLine.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ public static void Process(ref string[] args, ref LogLevel defaultLogLevel, out
2222
}
2323
else if (args[i] == "--skip-private-repositories")
2424
skipPrivateRepositories = true;
25+
else if (args[i] == "--inject")
26+
skipPrivateRepositories = true;
2527
else
2628
newArgs.Add(args[i]);
2729
}

‎src/tooling/Elastic.Documentation.Tooling/ExternalCommands/ExternalCommandExecutor.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,8 @@ string CaptureOutput()
112112
WorkingDirectory = workingDirectory.FullName,
113113
Timeout = TimeSpan.FromSeconds(3),
114114
WaitForExit = TimeSpan.FromSeconds(3),
115-
ConsoleOutWriter = NoopConsoleWriter.Instance
115+
ConsoleOutWriter = NoopConsoleWriter.Instance,
116+
OnlyPrintBinaryInExceptionMessage = false
116117
};
117118
var result = Proc.Start(arguments);
118119
var line = (result.ExitCode, muteExceptions) switch

‎src/tooling/docs-assembler/AssembleSources.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,7 @@ static void ReadHistoryMappings(IDictionary<string, IReadOnlyCollection<string>>
171171
public static FrozenDictionary<Uri, NavigationTocMapping> GetTocMappings(AssembleContext context)
172172
{
173173
var dictionary = new Dictionary<Uri, NavigationTocMapping>();
174-
var file = context.ConfigurationFileProvider.CreateNavigationFile(context.Configuration.PrivateRepositories);
174+
var file = context.ConfigurationFileProvider.CreateNavigationFile(context.Configuration);
175175
var reader = new YamlStreamReader(file, context.Collector);
176176
var entries = new List<KeyValuePair<Uri, NavigationTocMapping>>();
177177
foreach (var entry in reader.Read())

‎src/tooling/docs-assembler/Navigation/GlobalNavigationFile.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ public GlobalNavigationFile(AssembleContext context, AssembleSources assembleSou
2828
{
2929
_context = context;
3030
_assembleSources = assembleSources;
31-
NavigationFile = context.ConfigurationFileProvider.CreateNavigationFile(context.Configuration.PrivateRepositories);
31+
NavigationFile = context.ConfigurationFileProvider.CreateNavigationFile(context.Configuration);
3232
TableOfContents = Deserialize("toc");
3333
Phantoms = Deserialize("phantoms");
3434
ScopeDirectory = NavigationFile.Directory!;
@@ -63,7 +63,7 @@ public static ImmutableHashSet<Uri> GetPhantomPrefixes(AssembleContext context)
6363

6464
private static ImmutableHashSet<Uri> GetSourceUris(string key, AssembleContext context)
6565
{
66-
var navigationFile = context.ConfigurationFileProvider.CreateNavigationFile(context.Configuration.PrivateRepositories);
66+
var navigationFile = context.ConfigurationFileProvider.CreateNavigationFile(context.Configuration);
6767
var reader = new YamlStreamReader(navigationFile, context.Collector);
6868
var set = new HashSet<Uri>();
6969
foreach (var entry in reader.Read())
@@ -153,7 +153,7 @@ public void EmitError(string message) =>
153153

154154
private IReadOnlyCollection<TocReference> Deserialize(string key)
155155
{
156-
var navigationFile = _context.ConfigurationFileProvider.CreateNavigationFile(_context.Configuration.PrivateRepositories);
156+
var navigationFile = _context.ConfigurationFileProvider.CreateNavigationFile(_context.Configuration);
157157
var reader = new YamlStreamReader(navigationFile, _context.Collector);
158158
try
159159
{

‎src/tooling/docs-assembler/Sourcing/RepositorySourcesFetcher.cs‎

Lines changed: 26 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,12 @@ public CheckoutResult GetAll()
3535
foreach (var repo in repositories.Values)
3636
{
3737
var checkoutFolder = fs.DirectoryInfo.New(Path.Combine(context.CheckoutDirectory.FullName, repo.Name));
38+
// if we are running locally, allow for repository path override
39+
if (string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("CI")) && !string.IsNullOrWhiteSpace(repo.Path))
40+
{
41+
_logger.LogInformation("{RepositoryName}: Using local override path for {RepositoryName} at {Path}", repo.Name, repo.Name, repo.Path);
42+
checkoutFolder = fs.DirectoryInfo.New(repo.Path);
43+
}
3844
IGitRepository gitFacade = new SingleCommitOptimizedGitRepository(context.Collector, checkoutFolder);
3945
if (!checkoutFolder.Exists)
4046
{
@@ -130,30 +136,37 @@ public class RepositorySourcer(ILoggerFactory logFactory, IDirectoryInfo checkou
130136
public Checkout CloneRef(Repository repository, string gitRef, bool pull = false, int attempt = 1, bool appendRepositoryName = true, bool assumeCloned = false)
131137
{
132138
var checkoutFolder =
133-
appendRepositoryName
139+
appendRepositoryName
134140
? readFileSystem.DirectoryInfo.New(Path.Combine(checkoutDirectory.FullName, repository.Name))
135141
: checkoutDirectory;
142+
143+
// if we are running locally, allow for repository path override
144+
if (!string.IsNullOrWhiteSpace(repository.Path))
145+
{
146+
var di = readFileSystem.DirectoryInfo.New(repository.Path);
147+
if (!di.Exists)
148+
{
149+
_logger.LogInformation("{RepositoryName}: Can not find {RepositoryName}@{Commit} at local override path {CheckoutFolder}", repository.Name, repository.Name, gitRef, di.FullName);
150+
collector.EmitError("", $"Can not find {repository.Name}@{gitRef} at local override path {di.FullName}");
151+
return new Checkout { Directory = di, HeadReference = "", Repository = repository };
152+
}
153+
checkoutFolder = di;
154+
assumeCloned = true;
155+
_logger.LogInformation("{RepositoryName}: Using override path for {RepositoryName}@{Commit} at {CheckoutFolder}", repository.Name, repository.Name, gitRef, checkoutFolder.FullName);
156+
}
157+
136158
IGitRepository git = new SingleCommitOptimizedGitRepository(collector, checkoutFolder);
159+
137160
if (assumeCloned && checkoutFolder.Exists)
138161
{
139162
_logger.LogInformation("{RepositoryName}: Assuming {RepositoryName}@{Commit} is already checked out to {CheckoutFolder}", repository.Name, repository.Name, gitRef, checkoutFolder.FullName);
140-
return new Checkout
141-
{
142-
Directory = checkoutFolder,
143-
HeadReference = git.GetCurrentCommit(),
144-
Repository = repository,
145-
};
163+
return new Checkout { Directory = checkoutFolder, HeadReference = git.GetCurrentCommit(), Repository = repository };
146164
}
147165

148166
if (attempt > 3)
149167
{
150168
collector.EmitError("", $"Failed to clone repository {repository.Name}@{gitRef} after 3 attempts");
151-
return new Checkout
152-
{
153-
Directory = checkoutFolder,
154-
HeadReference = "",
155-
Repository = repository,
156-
};
169+
return new Checkout { Directory = checkoutFolder, HeadReference = "", Repository = repository };
157170
}
158171
_logger.LogInformation("{RepositoryName}: Cloning repository {RepositoryName}@{Commit} to {CheckoutFolder}", repository.Name, repository.Name, gitRef,
159172
checkoutFolder.FullName);

‎tests-integration/Elastic.Assembler.IntegrationTests/AssembleFixture.cs‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,12 @@
33
// See the LICENSE file in the project root for more information
44

55
using Aspire.Hosting;
6+
using Aspire.Hosting.ApplicationModel;
67
using Aspire.Hosting.Testing;
78
using Elastic.Documentation.ServiceDefaults;
89
using FluentAssertions;
910
using InMemLogger;
11+
using Microsoft.AspNetCore.Mvc.TagHelpers.Cache;
1012
using Microsoft.Extensions.DependencyInjection;
1113
using Microsoft.Extensions.Logging;
1214
using static Elastic.Documentation.Aspire.ResourceNames;
@@ -17,6 +19,7 @@ namespace Elastic.Assembler.IntegrationTests;
1719

1820
public class DocumentationFixture : IAsyncLifetime
1921
{
22+
private bool _failedBootstrap;
2023
public DistributedApplication DistributedApplication { get; private set; } = null!;
2124

2225
public InMemoryLogger InMemoryLogger { get; private set; } = null!;
@@ -39,9 +42,36 @@ public async ValueTask InitializeAsync()
3942
DistributedApplication = await builder.BuildAsync();
4043
InMemoryLogger = DistributedApplication.Services.GetService<InMemoryLogger>()!;
4144
_ = DistributedApplication.StartAsync().WaitAsync(TimeSpan.FromMinutes(5), TestContext.Current.CancellationToken);
45+
4246
_ = await DistributedApplication.ResourceNotifications
43-
.WaitForResourceHealthyAsync(AssemblerServe, cancellationToken: TestContext.Current.CancellationToken)
47+
.WaitForResourceAsync(AssemblerClone, KnownResourceStates.TerminalStates, cancellationToken: TestContext.Current.CancellationToken)
4448
.WaitAsync(TimeSpan.FromMinutes(5), TestContext.Current.CancellationToken);
49+
50+
await ValidateExitCode(AssemblerClone);
51+
52+
_ = await DistributedApplication.ResourceNotifications
53+
.WaitForResourceAsync(AssemblerBuild, KnownResourceStates.TerminalStates, cancellationToken: TestContext.Current.CancellationToken)
54+
.WaitAsync(TimeSpan.FromMinutes(5), TestContext.Current.CancellationToken);
55+
56+
await ValidateExitCode(AssemblerBuild);
57+
58+
_ = await DistributedApplication.ResourceNotifications
59+
.WaitForResourceHealthyAsync(AssemblerServe, cancellationToken: TestContext.Current.CancellationToken)
60+
.WaitAsync(TimeSpan.FromMinutes(1), TestContext.Current.CancellationToken);
61+
}
62+
63+
private async ValueTask ValidateExitCode(string resourceName)
64+
{
65+
var eventResource = await DistributedApplication.ResourceNotifications.WaitForResourceAsync(resourceName, _ => true);
66+
var id = eventResource.ResourceId;
67+
if (!DistributedApplication.ResourceNotifications.TryGetCurrentState(id, out var e))
68+
throw new Exception($"Could not find {resourceName} in the current state");
69+
if (e.Snapshot.ExitCode is not 0)
70+
{
71+
await DistributedApplication.StopAsync();
72+
await DistributedApplication.DisposeAsync();
73+
throw new Exception($"Exit code should be 0 for {resourceName}: {string.Join(Environment.NewLine, InMemoryLogger.RecordedLogs.Reverse().Take(30).Reverse())}");
74+
}
4575
}
4676

4777
/// <inheritdoc />

0 commit comments

Comments
 (0)