diff --git a/config/assembler.yml b/config/assembler.yml index e51c59dc5..4f8c4dc2b 100644 --- a/config/assembler.yml +++ b/config/assembler.yml @@ -136,3 +136,4 @@ references: current: 9.0 next: main skip: true + sparse_paths: ["docs", "config"] diff --git a/src/Elastic.Documentation.Configuration/Assembler/Repository.cs b/src/Elastic.Documentation.Configuration/Assembler/Repository.cs index a4150131c..cabdbf837 100644 --- a/src/Elastic.Documentation.Configuration/Assembler/Repository.cs +++ b/src/Elastic.Documentation.Configuration/Assembler/Repository.cs @@ -2,6 +2,7 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using System.Collections; using System.Runtime.Serialization; using YamlDotNet.Serialization; @@ -44,6 +45,9 @@ public record Repository [YamlMember(Alias = "edge")] public string GitReferenceEdge { get; set; } = "main"; + [YamlMember(Alias = "sparse_paths")] + public string[] SparsePaths { get; set; } = ["docs"]; + public string GetBranch(ContentSource contentSource) => contentSource switch { ContentSource.Current => GitReferenceCurrent, diff --git a/src/Elastic.Documentation.Configuration/ConfigurationFileProvider.cs b/src/Elastic.Documentation.Configuration/ConfigurationFileProvider.cs index 1f3d9f15d..31cb19e68 100644 --- a/src/Elastic.Documentation.Configuration/ConfigurationFileProvider.cs +++ b/src/Elastic.Documentation.Configuration/ConfigurationFileProvider.cs @@ -4,14 +4,26 @@ using System.IO.Abstractions; using Microsoft.Extensions.DependencyInjection; +using NetEscapades.EnumGenerators; namespace Elastic.Documentation.Configuration; +[EnumExtensions] +public enum ConfigurationSource +{ + Local, + Checkout, + Embedded +} + public class ConfigurationFileProvider { private readonly IFileSystem _fileSystem; private readonly string _assemblyName; + public ConfigurationSource ConfigurationSource { get; private set; } = ConfigurationSource.Embedded; + public string? GitReference { get; } + public ConfigurationFileProvider(IFileSystem fileSystem) { _fileSystem = fileSystem; @@ -22,6 +34,9 @@ public ConfigurationFileProvider(IFileSystem fileSystem) AssemblerFile = CreateTemporaryConfigurationFile("assembler.yml"); NavigationFile = CreateTemporaryConfigurationFile("navigation.yml"); LegacyUrlMappingsFile = CreateTemporaryConfigurationFile("legacy-url-mappings.yml"); + var path = GetAppDataPath("git-ref.txt"); + if (ConfigurationSource == ConfigurationSource.Checkout && _fileSystem.File.Exists(path)) + GitReference = _fileSystem.File.ReadAllText(path); } private IDirectoryInfo TemporaryDirectory { get; } @@ -45,11 +60,21 @@ private IFileInfo CreateTemporaryConfigurationFile(string fileName) private StreamReader GetLocalOrEmbedded(string fileName) { - var configPath = GetLocalPath(fileName); - if (!_fileSystem.File.Exists(configPath)) - return GetEmbeddedStream(fileName); - var reader = _fileSystem.File.OpenText(configPath); - return reader; + var localPath = GetLocalPath(fileName); + var appDataPath = GetAppDataPath(fileName); + if (_fileSystem.File.Exists(localPath)) + { + ConfigurationSource = ConfigurationSource.Local; + var reader = _fileSystem.File.OpenText(localPath); + return reader; + } + if (_fileSystem.File.Exists(appDataPath)) + { + ConfigurationSource = ConfigurationSource.Checkout; + var reader = _fileSystem.File.OpenText(appDataPath); + return reader; + } + return GetEmbeddedStream(fileName); } private StreamReader GetEmbeddedStream(string fileName) @@ -60,9 +85,11 @@ private StreamReader GetEmbeddedStream(string fileName) return reader; } - public static string LocalConfigurationDirectory => Path.Combine(Paths.WorkingDirectoryRoot.FullName, "config"); + private static string AppDataConfigurationDirectory { get; } = Path.Combine(Paths.ApplicationData.FullName, "config-clone", "config"); + private static string LocalConfigurationDirectory { get; } = Path.Combine(Paths.WorkingDirectoryRoot.FullName, "config"); private static string GetLocalPath(string file) => Path.Combine(LocalConfigurationDirectory, file); + private static string GetAppDataPath(string file) => Path.Combine(AppDataConfigurationDirectory, file); } public static class ConfigurationFileProviderServiceCollectionExtensions diff --git a/src/tooling/Elastic.Documentation.Tooling/Filters/InfoLoggerFilter.cs b/src/tooling/Elastic.Documentation.Tooling/Filters/InfoLoggerFilter.cs new file mode 100644 index 000000000..b6bc45ae9 --- /dev/null +++ b/src/tooling/Elastic.Documentation.Tooling/Filters/InfoLoggerFilter.cs @@ -0,0 +1,24 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Reflection; +using ConsoleAppFramework; +using Elastic.Documentation.Configuration; +using Microsoft.Extensions.Logging; + +namespace Elastic.Documentation.Tooling.Filters; + +public class InfoLoggerFilter(ConsoleAppFilter next, ILogger logger, ConfigurationFileProvider fileProvider) : ConsoleAppFilter(next) +{ + public override async Task InvokeAsync(ConsoleAppContext context, Cancel cancellationToken) + { + logger.LogInformation("Configuration source: {ConfigurationSource}", fileProvider.ConfigurationSource.ToStringFast(true)); + if (fileProvider.ConfigurationSource == ConfigurationSource.Checkout) + logger.LogInformation("Configuration source git reference: {ConfigurationSourceGitReference}", fileProvider.GitReference); + var assemblyVersion = Assembly.GetExecutingAssembly().GetCustomAttributes() + .FirstOrDefault()?.InformationalVersion; + logger.LogInformation("Version: {Version}", assemblyVersion); + await Next.InvokeAsync(context, cancellationToken); + } +} diff --git a/src/tooling/docs-assembler/Cli/RepositoryCommands.cs b/src/tooling/docs-assembler/Cli/RepositoryCommands.cs index ce2d9d3aa..e68b62689 100644 --- a/src/tooling/docs-assembler/Cli/RepositoryCommands.cs +++ b/src/tooling/docs-assembler/Cli/RepositoryCommands.cs @@ -42,6 +42,39 @@ private void AssignOutputLogger() ConsoleApp.LogError = msg => _log.LogError(msg); } + /// Clone the configuration folder + /// The git reference of the config, defaults to 'main' + [Command("init-config")] + public async Task CloneConfigurationFolder(string? gitRef = null, Cancel ctx = default) + { + await using var collector = new ConsoleDiagnosticsCollector(logFactory, githubActionsService).StartAsync(ctx); + + var fs = new FileSystem(); + var cachedPath = Path.Combine(Paths.ApplicationData.FullName, "config-clone"); + var checkoutFolder = fs.DirectoryInfo.New(cachedPath); + var cloner = new RepositorySourcer(logFactory, checkoutFolder, fs, collector); + + // relies on the embedded configuration, but we don't expect this to change + var repository = assemblyConfiguration.ReferenceRepositories["docs-builder"]; + repository = repository with + { + SparsePaths = ["config"] + }; + if (string.IsNullOrEmpty(gitRef)) + gitRef = "main"; + + _log.LogInformation("Cloning configuration ({GitReference})", gitRef); + var checkout = cloner.CloneRef(repository, gitRef, appendRepositoryName: false); + _log.LogInformation("Cloned configuration ({GitReference}) to {ConfigurationFolder}", checkout.HeadReference, checkout.Directory.FullName); + + var gitRefInformationFile = Path.Combine(cachedPath, "config", "git-ref.txt"); + await fs.File.WriteAllTextAsync(gitRefInformationFile, checkout.HeadReference, ctx); + + await collector.StopAsync(ctx); + return collector.Errors; + } + + /// Clones all repositories /// Treat warnings as errors and fail the build on warnings /// The environment to build diff --git a/src/tooling/docs-assembler/Program.cs b/src/tooling/docs-assembler/Program.cs index 449b46c89..bca2c65a7 100644 --- a/src/tooling/docs-assembler/Program.cs +++ b/src/tooling/docs-assembler/Program.cs @@ -21,6 +21,7 @@ var app = ConsoleApp.Create(); app.UseFilter(); +app.UseFilter(); app.UseFilter(); app.UseFilter(); diff --git a/src/tooling/docs-assembler/Sourcing/GitFacade.cs b/src/tooling/docs-assembler/Sourcing/GitFacade.cs index 6a518c3db..3526c0ae4 100644 --- a/src/tooling/docs-assembler/Sourcing/GitFacade.cs +++ b/src/tooling/docs-assembler/Sourcing/GitFacade.cs @@ -17,7 +17,7 @@ public interface IGitRepository bool IsInitialized(); void Pull(string branch); void Fetch(string reference); - void EnableSparseCheckout(string folder); + void EnableSparseCheckout(string[] folders); void DisableSparseCheckout(); void Checkout(string reference); } @@ -40,7 +40,7 @@ public class SingleCommitOptimizedGitRepository(DiagnosticsCollector collector, public bool IsInitialized() => Directory.Exists(Path.Combine(WorkingDirectory.FullName, ".git")); public void Pull(string branch) => ExecIn(EnvironmentVars, "git", "pull", "--depth", "1", "--allow-unrelated-histories", "--no-ff", "origin", branch); public void Fetch(string reference) => ExecIn(EnvironmentVars, "git", "fetch", "--no-tags", "--prune", "--no-recurse-submodules", "--depth", "1", "origin", reference); - public void EnableSparseCheckout(string folder) => ExecIn(EnvironmentVars, "git", "sparse-checkout", "set", folder); + public void EnableSparseCheckout(string[] folders) => ExecIn(EnvironmentVars, "git", ["sparse-checkout", "set", .. folders]); public void DisableSparseCheckout() => ExecIn(EnvironmentVars, "git", "sparse-checkout", "disable"); public void Checkout(string reference) => ExecIn(EnvironmentVars, "git", "checkout", "--force", reference); diff --git a/src/tooling/docs-assembler/Sourcing/RepositorySourcesFetcher.cs b/src/tooling/docs-assembler/Sourcing/RepositorySourcesFetcher.cs index 9dc3949f2..64cc5770d 100644 --- a/src/tooling/docs-assembler/Sourcing/RepositorySourcesFetcher.cs +++ b/src/tooling/docs-assembler/Sourcing/RepositorySourcesFetcher.cs @@ -130,9 +130,12 @@ public class RepositorySourcer(ILoggerFactory logFactory, IDirectoryInfo checkou // // The repository to clone. // The git reference to check out. Branch, commit or tag - public Checkout CloneRef(Repository repository, string gitRef, bool pull = false, int attempt = 1) + public Checkout CloneRef(Repository repository, string gitRef, bool pull = false, int attempt = 1, bool appendRepositoryName = true) { - var checkoutFolder = readFileSystem.DirectoryInfo.New(Path.Combine(checkoutDirectory.FullName, repository.Name)); + var checkoutFolder = + appendRepositoryName + ? readFileSystem.DirectoryInfo.New(Path.Combine(checkoutDirectory.FullName, repository.Name)) + : checkoutDirectory; IGitRepository git = new SingleCommitOptimizedGitRepository(collector, checkoutFolder); if (attempt > 3) { @@ -228,7 +231,7 @@ private static void FetchAndCheckout(IGitRepository git, Repository repository, git.DisableSparseCheckout(); break; case CheckoutStrategy.Partial: - git.EnableSparseCheckout("docs"); + git.EnableSparseCheckout(repository.SparsePaths); break; default: throw new ArgumentOutOfRangeException(nameof(repository), repository.CheckoutStrategy, null); diff --git a/src/tooling/docs-builder/Program.cs b/src/tooling/docs-builder/Program.cs index e823ca0ea..493064625 100644 --- a/src/tooling/docs-builder/Program.cs +++ b/src/tooling/docs-builder/Program.cs @@ -15,6 +15,7 @@ var app = ConsoleApp.Create(); app.UseFilter(); +app.UseFilter(); app.UseFilter(); app.UseFilter(); app.UseFilter();