feature/config folder arg (#1580)

* Ensure we assign record with with syntax to ensure we do not update shared instances of repository

* Add `repo init-config` command

This allows the build to source the configuration prior to cloning and building

docs-assembler repo init-config
docs-assembler repo clone-all
docs-assembler repo build-all

On production we can lock the configuration to a commit using `--git-ref sha`.

ConfigurationFileProvider will look at

- {pwd}/config, development
- {app_data}/config, (where init-config clones too)
- embedded sources

We also now log the configuration source and in case of app_data also the git reference.

Author: Mpdreamz

config/assembler.yml

@@ -136,3 +136,4 @@ references:
     current: 9.0
     next: main
     skip: true
+    sparse_paths: ["docs", "config"]

src/Elastic.Documentation.Configuration/Assembler/Repository.cs

@@ -2,6 +2,7 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System.Collections;
 using System.Runtime.Serialization;
 using YamlDotNet.Serialization;
 
@@ -44,6 +45,9 @@ public record Repository
 	[YamlMember(Alias = "edge")]
 	public string GitReferenceEdge { get; set; } = "main";
 
+	[YamlMember(Alias = "sparse_paths")]
+	public string[] SparsePaths { get; set; } = ["docs"];
+
 	public string GetBranch(ContentSource contentSource) => contentSource switch
 	{
 		ContentSource.Current => GitReferenceCurrent,

src/Elastic.Documentation.Configuration/ConfigurationFileProvider.cs

@@ -4,14 +4,26 @@
 
 using System.IO.Abstractions;
 using Microsoft.Extensions.DependencyInjection;
+using NetEscapades.EnumGenerators;
 
 namespace Elastic.Documentation.Configuration;
 
+[EnumExtensions]
+public enum ConfigurationSource
+{
+	Local,
+	Checkout,
+	Embedded
+}
+
 public class ConfigurationFileProvider
 {
 	private readonly IFileSystem _fileSystem;
 	private readonly string _assemblyName;
 
+	public ConfigurationSource ConfigurationSource { get; private set; } = ConfigurationSource.Embedded;
+	public string? GitReference { get; }
+
 	public ConfigurationFileProvider(IFileSystem fileSystem)
 	{
 		_fileSystem = fileSystem;
@@ -22,6 +34,9 @@ public ConfigurationFileProvider(IFileSystem fileSystem)
 		AssemblerFile = CreateTemporaryConfigurationFile("assembler.yml");
 		NavigationFile = CreateTemporaryConfigurationFile("navigation.yml");
 		LegacyUrlMappingsFile = CreateTemporaryConfigurationFile("legacy-url-mappings.yml");
+		var path = GetAppDataPath("git-ref.txt");
+		if (ConfigurationSource == ConfigurationSource.Checkout && _fileSystem.File.Exists(path))
+			GitReference = _fileSystem.File.ReadAllText(path);
 	}
 
 	private IDirectoryInfo TemporaryDirectory { get; }
@@ -45,11 +60,21 @@ private IFileInfo CreateTemporaryConfigurationFile(string fileName)
 
 	private StreamReader GetLocalOrEmbedded(string fileName)
 	{
-		var configPath = GetLocalPath(fileName);
-		if (!_fileSystem.File.Exists(configPath))
-			return GetEmbeddedStream(fileName);
-		var reader = _fileSystem.File.OpenText(configPath);
-		return reader;
+		var localPath = GetLocalPath(fileName);
+		var appDataPath = GetAppDataPath(fileName);
+		if (_fileSystem.File.Exists(localPath))
+		{
+			ConfigurationSource = ConfigurationSource.Local;
+			var reader = _fileSystem.File.OpenText(localPath);
+			return reader;
+		}
+		if (_fileSystem.File.Exists(appDataPath))
+		{
+			ConfigurationSource = ConfigurationSource.Checkout;
+			var reader = _fileSystem.File.OpenText(appDataPath);
+			return reader;
+		}
+		return GetEmbeddedStream(fileName);
 	}
 
 	private StreamReader GetEmbeddedStream(string fileName)
@@ -60,9 +85,11 @@ private StreamReader GetEmbeddedStream(string fileName)
 		return reader;
 	}
 
-	public static string LocalConfigurationDirectory => Path.Combine(Paths.WorkingDirectoryRoot.FullName, "config");
+	private static string AppDataConfigurationDirectory { get; } = Path.Combine(Paths.ApplicationData.FullName, "config-clone", "config");
+	private static string LocalConfigurationDirectory { get; } = Path.Combine(Paths.WorkingDirectoryRoot.FullName, "config");
 
 	private static string GetLocalPath(string file) => Path.Combine(LocalConfigurationDirectory, file);
+	private static string GetAppDataPath(string file) => Path.Combine(AppDataConfigurationDirectory, file);
 }
 
 public static class ConfigurationFileProviderServiceCollectionExtensions

src/tooling/Elastic.Documentation.Tooling/Filters/InfoLoggerFilter.cs

@@ -0,0 +1,24 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Reflection;
+using ConsoleAppFramework;
+using Elastic.Documentation.Configuration;
+using Microsoft.Extensions.Logging;
+
+namespace Elastic.Documentation.Tooling.Filters;
+
+public class InfoLoggerFilter(ConsoleAppFilter next, ILogger<InfoLoggerFilter> logger, ConfigurationFileProvider fileProvider) : ConsoleAppFilter(next)
+{
+	public override async Task InvokeAsync(ConsoleAppContext context, Cancel cancellationToken)
+	{
+		logger.LogInformation("Configuration source: {ConfigurationSource}", fileProvider.ConfigurationSource.ToStringFast(true));
+		if (fileProvider.ConfigurationSource == ConfigurationSource.Checkout)
+			logger.LogInformation("Configuration source git reference: {ConfigurationSourceGitReference}", fileProvider.GitReference);
+		var assemblyVersion = Assembly.GetExecutingAssembly().GetCustomAttributes<AssemblyInformationalVersionAttribute>()
+			.FirstOrDefault()?.InformationalVersion;
+		logger.LogInformation("Version: {Version}", assemblyVersion);
+		await Next.InvokeAsync(context, cancellationToken);
+	}
+}

src/tooling/docs-assembler/Cli/RepositoryCommands.cs

@@ -42,6 +42,39 @@ private void AssignOutputLogger()
 		ConsoleApp.LogError = msg => _log.LogError(msg);
 	}
 
+	/// <summary> Clone the configuration folder </summary>
+	/// <param name="gitRef">The git reference of the config, defaults to 'main'</param>
+	[Command("init-config")]
+	public async Task<int> CloneConfigurationFolder(string? gitRef = null, Cancel ctx = default)
+	{
+		await using var collector = new ConsoleDiagnosticsCollector(logFactory, githubActionsService).StartAsync(ctx);
+
+		var fs = new FileSystem();
+		var cachedPath = Path.Combine(Paths.ApplicationData.FullName, "config-clone");
+		var checkoutFolder = fs.DirectoryInfo.New(cachedPath);
+		var cloner = new RepositorySourcer(logFactory, checkoutFolder, fs, collector);
+
+		// relies on the embedded configuration, but we don't expect this to change
+		var repository = assemblyConfiguration.ReferenceRepositories["docs-builder"];
+		repository = repository with
+		{
+			SparsePaths = ["config"]
+		};
+		if (string.IsNullOrEmpty(gitRef))
+			gitRef = "main";
+
+		_log.LogInformation("Cloning configuration ({GitReference})", gitRef);
+		var checkout = cloner.CloneRef(repository, gitRef, appendRepositoryName: false);
+		_log.LogInformation("Cloned configuration ({GitReference}) to {ConfigurationFolder}", checkout.HeadReference, checkout.Directory.FullName);
+
+		var gitRefInformationFile = Path.Combine(cachedPath, "config", "git-ref.txt");
+		await fs.File.WriteAllTextAsync(gitRefInformationFile, checkout.HeadReference, ctx);
+
+		await collector.StopAsync(ctx);
+		return collector.Errors;
+	}
+
+
 	/// <summary> Clones all repositories </summary>
 	/// <param name="strict"> Treat warnings as errors and fail the build on warnings</param>
 	/// <param name="environment"> The environment to build</param>

src/tooling/docs-assembler/Program.cs

@@ -21,6 +21,7 @@
 
 var app = ConsoleApp.Create();
 app.UseFilter<ReplaceLogFilter>();
+app.UseFilter<InfoLoggerFilter>();
 app.UseFilter<StopwatchFilter>();
 app.UseFilter<CatchExceptionFilter>();
 

src/tooling/docs-assembler/Sourcing/GitFacade.cs

@@ -17,7 +17,7 @@ public interface IGitRepository
 	bool IsInitialized();
 	void Pull(string branch);
 	void Fetch(string reference);
-	void EnableSparseCheckout(string folder);
+	void EnableSparseCheckout(string[] folders);
 	void DisableSparseCheckout();
 	void Checkout(string reference);
 }
@@ -40,7 +40,7 @@ public class SingleCommitOptimizedGitRepository(DiagnosticsCollector collector,
 	public bool IsInitialized() => Directory.Exists(Path.Combine(WorkingDirectory.FullName, ".git"));
 	public void Pull(string branch) => ExecIn(EnvironmentVars, "git", "pull", "--depth", "1", "--allow-unrelated-histories", "--no-ff", "origin", branch);
 	public void Fetch(string reference) => ExecIn(EnvironmentVars, "git", "fetch", "--no-tags", "--prune", "--no-recurse-submodules", "--depth", "1", "origin", reference);
-	public void EnableSparseCheckout(string folder) => ExecIn(EnvironmentVars, "git", "sparse-checkout", "set", folder);
+	public void EnableSparseCheckout(string[] folders) => ExecIn(EnvironmentVars, "git", ["sparse-checkout", "set", .. folders]);
 	public void DisableSparseCheckout() => ExecIn(EnvironmentVars, "git", "sparse-checkout", "disable");
 	public void Checkout(string reference) => ExecIn(EnvironmentVars, "git", "checkout", "--force", reference);
 

src/tooling/docs-assembler/Sourcing/RepositorySourcesFetcher.cs

@@ -130,9 +130,12 @@ public class RepositorySourcer(ILoggerFactory logFactory, IDirectoryInfo checkou
 	// </summary>
 	// <param name="repository">The repository to clone.</param>
 	// <param name="gitRef">The git reference to check out. Branch, commit or tag</param>
-	public Checkout CloneRef(Repository repository, string gitRef, bool pull = false, int attempt = 1)
+	public Checkout CloneRef(Repository repository, string gitRef, bool pull = false, int attempt = 1, bool appendRepositoryName = true)
 	{
-		var checkoutFolder = readFileSystem.DirectoryInfo.New(Path.Combine(checkoutDirectory.FullName, repository.Name));
+		var checkoutFolder =
+			appendRepositoryName
+				? readFileSystem.DirectoryInfo.New(Path.Combine(checkoutDirectory.FullName, repository.Name))
+				: checkoutDirectory;
 		IGitRepository git = new SingleCommitOptimizedGitRepository(collector, checkoutFolder);
 		if (attempt > 3)
 		{
@@ -228,7 +231,7 @@ private static void FetchAndCheckout(IGitRepository git, Repository repository,
 				git.DisableSparseCheckout();
 				break;
 			case CheckoutStrategy.Partial:
-				git.EnableSparseCheckout("docs");
+				git.EnableSparseCheckout(repository.SparsePaths);
 				break;
 			default:
 				throw new ArgumentOutOfRangeException(nameof(repository), repository.CheckoutStrategy, null);

src/tooling/docs-builder/Program.cs

@@ -15,6 +15,7 @@
 var app = ConsoleApp.Create();
 
 app.UseFilter<ReplaceLogFilter>();
+app.UseFilter<InfoLoggerFilter>();
 app.UseFilter<StopwatchFilter>();
 app.UseFilter<CatchExceptionFilter>();
 app.UseFilter<CheckForUpdatesFilter>();