Skip to content

Commit f07e35d

Browse files
MpdreamzCopilot
Mpdreamz
and
Copilot
authored
Initial version of sourcing repositories for assembly (#713)
* Initial version of sourcing repositories for assembly * Apply suggestions from code review Co-authored-by: Copilot <[email protected]> * Use readfile system to check if directory exists --------- Co-authored-by: Copilot <[email protected]>
1 parent 299898f commit f07e35d

File tree

7 files changed

+228
-234
lines changed

7 files changed

+228
-234
lines changed

‎src/docs-assembler/AssembleContext.cs

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
using System.IO.Abstractions;
6+
using Documentation.Assembler.Configuration;
7+
using Elastic.Markdown.Diagnostics;
8+
using Elastic.Markdown.IO;
9+
10+
namespace Documentation.Assembler;
11+
12+
public class AssembleContext
13+
{
14+
public IFileSystem ReadFileSystem { get; }
15+
public IFileSystem WriteFileSystem { get; }
16+
17+
public DiagnosticsCollector Collector { get; }
18+
19+
public AssemblyConfiguration Configuration { get; set; }
20+
21+
public IFileInfo ConfigurationPath { get; }
22+
23+
public IDirectoryInfo OutputDirectory { get; set; }
24+
25+
public AssembleContext(DiagnosticsCollector collector, IFileSystem readFileSystem, IFileSystem writeFileSystem, string? output)
26+
{
27+
Collector = collector;
28+
ReadFileSystem = readFileSystem;
29+
WriteFileSystem = writeFileSystem;
30+
31+
var configPath = Path.Combine(Paths.Root.FullName, "src/docs-assembler/assembler.yml");
32+
ConfigurationPath = ReadFileSystem.FileInfo.New(configPath);
33+
Configuration = AssemblyConfiguration.Deserialize(ReadFileSystem.File.ReadAllText(ConfigurationPath.FullName));
34+
OutputDirectory = ReadFileSystem.DirectoryInfo.New(output ?? ".artifacts/assembly");
35+
}
36+
}

‎src/docs-assembler/Cli/RepositoryCommands.cs

Lines changed: 9 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -2,27 +2,17 @@
22
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
33
// See the LICENSE file in the project root for more information
44

5-
using System.Collections.Concurrent;
6-
using System.Diagnostics;
75
using System.Diagnostics.CodeAnalysis;
6+
using System.IO.Abstractions;
7+
using Actions.Core.Services;
88
using ConsoleAppFramework;
9-
using Elastic.Markdown.IO;
9+
using Documentation.Assembler.Sourcing;
10+
using Elastic.Documentation.Tooling.Diagnostics.Console;
1011
using Microsoft.Extensions.Logging;
11-
using ProcNet;
12-
using ProcNet.Std;
1312

1413
namespace Documentation.Assembler.Cli;
1514

16-
public class ConsoleLineHandler(string prefix) : IConsoleLineHandler
17-
{
18-
public void Handle(LineOut lineOut) => lineOut.CharsOrString(
19-
r => Console.Write(prefix + ": " + r),
20-
l => Console.WriteLine(prefix + ": " + l));
21-
22-
public void Handle(Exception e) { }
23-
}
24-
25-
internal sealed class RepositoryCommands(ILoggerFactory logger)
15+
internal sealed class RepositoryCommands(ICoreService githubActionsService, ILoggerFactory logger)
2616
{
2717
[SuppressMessage("Usage", "CA2254:Template should be a static expression")]
2818
private void AssignOutputLogger()
@@ -40,58 +30,13 @@ private void AssignOutputLogger()
4030
public async Task CloneAll(Cancel ctx = default)
4131
{
4232
AssignOutputLogger();
43-
var configFile = Path.Combine(Paths.Root.FullName, "src/docs-assembler/conf.yml");
44-
var config = AssemblyConfiguration.Deserialize(File.ReadAllText(configFile));
45-
46-
Console.WriteLine(config.Repositories.Count);
47-
var dict = new ConcurrentDictionary<string, Stopwatch>();
48-
await Parallel.ForEachAsync(config.Repositories,
49-
new ParallelOptions { CancellationToken = ctx, MaxDegreeOfParallelism = Environment.ProcessorCount / 4 }, async (kv, c) =>
50-
{
51-
await Task.Run(() =>
52-
{
53-
var name = kv.Key;
54-
var repository = kv.Value;
55-
var checkoutFolder = Path.Combine(Paths.Root.FullName, $".artifacts/assembly/{name}");
5633

57-
var sw = Stopwatch.StartNew();
58-
_ = dict.AddOrUpdate(name, sw, (_, _) => sw);
59-
Console.WriteLine($"Checkout: {name}\t{repository}\t{checkoutFolder}");
60-
var branch = repository.Branch ?? "main";
61-
var args = new StartArguments(
62-
"git", "clone", repository.Origin, checkoutFolder, "--depth", "1"
63-
, "--single-branch", "--branch", branch
64-
);
65-
_ = Proc.StartRedirected(args, new ConsoleLineHandler(name));
66-
sw.Stop();
67-
}, c);
68-
}).ConfigureAwait(false);
34+
await using var collector = new ConsoleDiagnosticsCollector(logger, githubActionsService);
6935

70-
foreach (var kv in dict.OrderBy(kv => kv.Value.Elapsed))
71-
Console.WriteLine($"-> {kv.Key}\ttook: {kv.Value.Elapsed}");
36+
var assembleContext = new AssembleContext(collector, new FileSystem(), new FileSystem(), null);
37+
var cloner = new RepositoryCloner(logger, assembleContext);
38+
await cloner.CloneAll(ctx);
7239
}
7340

74-
/// <summary> List all checked out repositories </summary>
75-
[Command("list")]
76-
public async Task ListRepositories()
77-
{
78-
AssignOutputLogger();
79-
var assemblyPath = Path.Combine(Paths.Root.FullName, $".artifacts/assembly");
80-
var dir = new DirectoryInfo(assemblyPath);
81-
var dictionary = new Dictionary<string, string>();
82-
foreach (var d in dir.GetDirectories())
83-
{
84-
var checkoutFolder = Path.Combine(assemblyPath, d.Name);
85-
86-
var capture = Proc.Start(
87-
new StartArguments("git", "rev-parse", "--abbrev-ref", "HEAD") { WorkingDirectory = checkoutFolder }
88-
);
89-
dictionary.Add(d.Name, capture.ConsoleOut.FirstOrDefault()?.Line ?? "unknown");
90-
}
9141

92-
foreach (var kv in dictionary.OrderBy(kv => kv.Value))
93-
Console.WriteLine($"-> {kv.Key}\tbranch: {kv.Value}");
94-
95-
await Task.CompletedTask;
96-
}
9742
}

‎src/docs-assembler/AssemblyConfiguration.cs renamed to ‎src/docs-assembler/Configuration/AssemblyConfiguration.cs

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,12 @@
44

55
using YamlDotNet.Serialization;
66

7-
namespace Documentation.Assembler;
7+
namespace Documentation.Assembler.Configuration;
88

99
[YamlStaticContext]
1010
[YamlSerializable(typeof(AssemblyConfiguration))]
1111
[YamlSerializable(typeof(Repository))]
12+
[YamlSerializable(typeof(NarrativeRepository))]
1213
public partial class YamlStaticContext;
1314

1415
public record AssemblyConfiguration
@@ -34,16 +35,27 @@ public static AssemblyConfiguration Deserialize(string yaml)
3435
}
3536
}
3637

37-
[YamlMember(Alias = "repos")]
38-
public Dictionary<string, Repository> Repositories { get; set; } = [];
38+
[YamlMember(Alias = "narrative")]
39+
public NarrativeRepository Narrative { get; set; } = new();
40+
41+
[YamlMember(Alias = "references")]
42+
public Dictionary<string, Repository?> ReferenceRepositories { get; set; } = [];
43+
}
44+
45+
public record NarrativeRepository : Repository
46+
{
47+
public static string Name { get; } = "docs-content";
3948
}
4049

4150
public record Repository
4251
{
4352
[YamlMember(Alias = "repo")]
44-
public string Origin { get; set; } = string.Empty;
53+
public string? Origin { get; set; }
54+
55+
[YamlMember(Alias = "current")]
56+
public string? CurrentBranch { get; set; }
4557

46-
[YamlMember(Alias = "branch")]
47-
public string? Branch { get; set; }
58+
[YamlMember(Alias = "checkout_strategy")]
59+
public string CheckoutStrategy { get; set; } = "partial";
4860

4961
}

‎src/docs-assembler/Sourcing/RepositoryCloner.cs

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
using System.Collections.Concurrent;
6+
using System.Diagnostics;
7+
using Documentation.Assembler.Configuration;
8+
using Elastic.Markdown.Diagnostics;
9+
using Elastic.Markdown.IO;
10+
using Microsoft.Extensions.Logging;
11+
using ProcNet;
12+
using ProcNet.Std;
13+
14+
namespace Documentation.Assembler.Sourcing;
15+
16+
public class RepositoryCloner(ILoggerFactory logger, AssembleContext context)
17+
{
18+
private readonly ILogger<RepositoryCloner> _logger = logger.CreateLogger<RepositoryCloner>();
19+
20+
private AssemblyConfiguration Configuration => context.Configuration;
21+
22+
public async Task CloneAll(Cancel ctx = default)
23+
{
24+
var dict = new ConcurrentDictionary<string, Stopwatch>();
25+
26+
_logger.LogInformation("Cloning narrative content: {Repository}", NarrativeRepository.Name);
27+
CloneRepository(Configuration.Narrative, NarrativeRepository.Name, dict);
28+
29+
_logger.LogInformation("Cloning {ReferenceRepositoryCount} repositories", Configuration.ReferenceRepositories.Count);
30+
await Parallel.ForEachAsync(Configuration.ReferenceRepositories,
31+
new ParallelOptions
32+
{
33+
CancellationToken = ctx,
34+
MaxDegreeOfParallelism = Environment.ProcessorCount
35+
}, async (kv, c) =>
36+
{
37+
await Task.Run(() =>
38+
{
39+
var name = kv.Key.Trim();
40+
CloneRepository(kv.Value, name, dict);
41+
}, c);
42+
}).ConfigureAwait(false);
43+
44+
foreach (var kv in dict.OrderBy(kv => kv.Value.Elapsed))
45+
Console.WriteLine($"-> {kv.Key}\ttook: {kv.Value.Elapsed}");
46+
}
47+
48+
private void CloneRepository(Repository? repository, string name, ConcurrentDictionary<string, Stopwatch> dict)
49+
{
50+
repository ??= new Repository();
51+
repository.CurrentBranch ??= "main";
52+
repository.Origin ??= $"[email protected]:elastic/{name}.git";
53+
54+
var checkoutFolder = Path.Combine(context.OutputDirectory.FullName, name);
55+
var relativePath = Path.GetRelativePath(Paths.Root.FullName, checkoutFolder);
56+
var sw = Stopwatch.StartNew();
57+
_ = dict.AddOrUpdate(name, sw, (_, _) => sw);
58+
if (context.ReadFileSystem.Directory.Exists(checkoutFolder))
59+
{
60+
_logger.LogInformation("Pull: {Name}\t{Repository}\t{RelativePath}", name, repository, relativePath);
61+
// --allow-unrelated-histories due to shallow clones not finding a common ancestor
62+
ExecIn(checkoutFolder, "git", "pull", "--depth", "1", "--allow-unrelated-histories", "--no-ff");
63+
}
64+
else
65+
{
66+
_logger.LogInformation("Checkout: {Name}\t{Repository}\t{RelativePath}", name, repository, relativePath);
67+
if (repository.CheckoutStrategy == "full")
68+
{
69+
Exec("git", "clone", repository.Origin, checkoutFolder,
70+
"--depth", "1", "--single-branch",
71+
"--branch", repository.CurrentBranch
72+
);
73+
}
74+
else if (repository.CheckoutStrategy == "partial")
75+
{
76+
Exec(
77+
"git", "clone", "--filter=blob:none", "--no-checkout", repository.Origin, checkoutFolder
78+
);
79+
80+
ExecIn(checkoutFolder, "git", "sparse-checkout", "set", "--cone");
81+
ExecIn(checkoutFolder, "git", "checkout", repository.CurrentBranch);
82+
ExecIn(checkoutFolder, "git", "sparse-checkout", "set", "docs");
83+
}
84+
}
85+
86+
sw.Stop();
87+
88+
void Exec(string binary, params string[] args) => ExecIn(null, binary, args);
89+
90+
void ExecIn(string? workingDirectory, string binary, params string[] args)
91+
{
92+
var arguments = new StartArguments(binary, args)
93+
{
94+
WorkingDirectory = workingDirectory
95+
};
96+
var result = Proc.StartRedirected(arguments, new ConsoleLineHandler(_logger, name));
97+
if (result.ExitCode != 0)
98+
context.Collector.EmitError("", $"Exit code: {result.ExitCode} while executing {binary} {string.Join(" ", arguments)}");
99+
}
100+
}
101+
}
102+
103+
public class ConsoleLineHandler(ILogger<RepositoryCloner> logger, string prefix) : IConsoleLineHandler
104+
{
105+
public void Handle(LineOut lineOut) => lineOut.CharsOrString(
106+
r => Console.Write(prefix + ": " + r),
107+
l => logger.LogInformation("{RepositoryName}: {Message}", prefix, l)
108+
);
109+
110+
public void Handle(Exception e) { }
111+
}

‎src/docs-assembler/assembler.yml

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
narrative:
2+
checkout_strategy: full
3+
references:
4+
apm-agent-android:
5+
apm-agent-dotnet:
6+
apm-agent-go:
7+
apm-agent-ios:
8+
apm-agent-java:
9+
apm-agent-nodejs:
10+
apm-agent-php:
11+
apm-agent-python:
12+
apm-agent-ruby:
13+
apm-agent-rum-js:
14+
apm-aws-lambda:
15+
apm-k8s-attacher:
16+
beats:
17+
cloud-on-k8s:
18+
cloud:
19+
current: master
20+
curator:
21+
current: master
22+
ecctl:
23+
current: master
24+
ecs-dotnet:
25+
ecs-logging-go-logrus:
26+
ecs-logging-go-zap:
27+
ecs-logging-go-zerolog:
28+
ecs-logging-java:
29+
ecs-logging-nodejs:
30+
ecs-logging-php:
31+
ecs-logging-python:
32+
ecs-logging-ruby:
33+
ecs-logging:
34+
ecs:
35+
eland:
36+
elastic-serverless-forwarder:
37+
elasticsearch-hadoop:
38+
elasticsearch-java:
39+
elasticsearch-js:
40+
elasticsearch-net:
41+
elasticsearch-php:
42+
elasticsearch-py:
43+
elasticsearch-rs:
44+
elasticsearch-ruby:
45+
elasticsearch:
46+
go-elasticsearch:
47+
integrations:
48+
kibana:
49+
logstash-docs:
50+
logstash:
51+
search-ui:
52+
integration-docs:
53+
security-docs:

0 commit comments

Comments
 (0)