Support building only public repositories

Author: Mpdreamz

config/assembler.yml

@@ -94,6 +94,7 @@ references:
   elastic-otel-python:
   elastic-serverless-forwarder:
   integration-docs:
+    private: true
   integrations:
   logstash-docs-md:
   opentelemetry:
@@ -104,7 +105,12 @@ references:
 
   # @elastic/admin-docs
   cloud-on-k8s:
-  cloud: *master
+  cloud:
+    current: master
+    next: master
+    edge: master
+    private: true
+
   curator: *master
   ecctl: *master
 

src/Elastic.Documentation.Configuration/Assembler/AssemblyConfiguration.cs

@@ -4,7 +4,6 @@
 
 using System.Text.RegularExpressions;
 using Elastic.Documentation.Extensions;
-using YamlDotNet.RepresentationModel;
 using YamlDotNet.Serialization;
 using YamlStaticContext = Elastic.Documentation.Configuration.Serialization.YamlStaticContext;
 
@@ -13,9 +12,9 @@ namespace Elastic.Documentation.Configuration.Assembler;
 public record AssemblyConfiguration
 {
 	public static AssemblyConfiguration Create(ConfigurationFileProvider provider) =>
-		Deserialize(provider.AssemblerFile.ReadToEnd());
+		Deserialize(provider.AssemblerFile.ReadToEnd(), skipPrivateRepositories: provider.SkipPrivateRepositories);
 
-	public static AssemblyConfiguration Deserialize(string yaml)
+	public static AssemblyConfiguration Deserialize(string yaml, bool skipPrivateRepositories = false)
 	{
 		var input = new StringReader(yaml);
 
@@ -28,13 +27,28 @@ public static AssemblyConfiguration Deserialize(string yaml)
 			var config = deserializer.Deserialize<AssemblyConfiguration>(input);
 			foreach (var (name, r) in config.ReferenceRepositories)
 			{
+				if (name == "cloud")
+				{
+				}
 				var repository = RepositoryDefaults(r, name);
 				config.ReferenceRepositories[name] = repository;
 			}
+			var privateRepositories = config.ReferenceRepositories.Where(r => r.Value.Private).ToList();
+			foreach (var (name, _) in privateRepositories)
+			{
+				if (skipPrivateRepositories)
+					_ = config.ReferenceRepositories.Remove(name);
+			}
 
 			foreach (var (name, env) in config.Environments)
 				env.Name = name;
 			config.Narrative = RepositoryDefaults(config.Narrative, NarrativeRepository.RepositoryName);
+
+			config.AvailableRepositories = config.ReferenceRepositories.Values
+				.Where(r => !r.Skip)
+				.Concat([config.Narrative]).ToDictionary(kvp => kvp.Name, kvp => kvp);
+
+			config.PrivateRepositories = privateRepositories.ToDictionary(kvp => kvp.Key, kvp => kvp.Value);
 			return config;
 		}
 		catch (Exception e)
@@ -80,6 +94,15 @@ private static TRepository RepositoryDefaults<TRepository>(TRepository r, string
 	[YamlMember(Alias = "references")]
 	public Dictionary<string, Repository> ReferenceRepositories { get; set; } = [];
 
+	/// All available repositories, combines <see cref="ReferenceRepositories"/> and <see cref="Narrative"/> and will filter private repositories if `skip-private-repositories`
+	/// is specified
+	[YamlIgnore]
+	public IReadOnlyDictionary<string, Repository> AvailableRepositories { get; private set; } = new Dictionary<string, Repository>();
+
+	/// Repositories marked as private, these are listed under <see cref="AvailableRepositories"/> if `--skip-private-repositories` is not specified
+	[YamlIgnore]
+	public IReadOnlyDictionary<string, Repository> PrivateRepositories { get; private set; } = new Dictionary<string, Repository>();
+
 	[YamlMember(Alias = "environments")]
 	public Dictionary<string, PublishEnvironment> Environments { get; set; } = [];
 

src/Elastic.Documentation.Configuration/Assembler/Repository.cs

@@ -48,6 +48,9 @@ public record Repository
 	[YamlMember(Alias = "sparse_paths")]
 	public string[] SparsePaths { get; set; } = ["docs"];
 
+	[YamlMember(Alias = "private")]
+	public bool Private { get; set; }
+
 	public string GetBranch(ContentSource contentSource) => contentSource switch
 	{
 		ContentSource.Current => GitReferenceCurrent,

src/Elastic.Documentation.Configuration/ConfigurationFileProvider.cs

@@ -3,8 +3,11 @@
 // See the LICENSE file in the project root for more information
 
 using System.IO.Abstractions;
+using System.Text.RegularExpressions;
+using Elastic.Documentation.Configuration.Assembler;
 using Microsoft.Extensions.DependencyInjection;
 using NetEscapades.EnumGenerators;
+using YamlDotNet.RepresentationModel;
 
 namespace Elastic.Documentation.Configuration;
 
@@ -16,18 +19,19 @@ public enum ConfigurationSource
 	Embedded
 }
 
-public class ConfigurationFileProvider
+public partial class ConfigurationFileProvider
 {
 	private readonly IFileSystem _fileSystem;
 	private readonly string _assemblyName;
 
 	public ConfigurationSource ConfigurationSource { get; private set; } = ConfigurationSource.Embedded;
 	public string? GitReference { get; }
 
-	public ConfigurationFileProvider(IFileSystem fileSystem)
+	public ConfigurationFileProvider(IFileSystem fileSystem, bool skipPrivateRepositories = false)
 	{
 		_fileSystem = fileSystem;
 		_assemblyName = typeof(ConfigurationFileProvider).Assembly.GetName().Name!;
+		SkipPrivateRepositories = skipPrivateRepositories;
 		TemporaryDirectory = fileSystem.Directory.CreateTempSubdirectory("docs-builder-config");
 
 		VersionFile = CreateTemporaryConfigurationFile("versions.yml");
@@ -39,16 +43,75 @@ public ConfigurationFileProvider(IFileSystem fileSystem)
 			GitReference = _fileSystem.File.ReadAllText(path);
 	}
 
+	public bool SkipPrivateRepositories { get; }
+
 	private IDirectoryInfo TemporaryDirectory { get; }
 
-	public IFileInfo NavigationFile { get; }
+	public IFileInfo NavigationFile { get; private set; }
 
 	public IFileInfo VersionFile { get; }
 
 	public IFileInfo AssemblerFile { get; }
 
 	public IFileInfo LegacyUrlMappingsFile { get; }
 
+	public IFileInfo CreateNavigationFile(IReadOnlyDictionary<string, Repository> privateRepositories)
+	{
+		if (privateRepositories.Count == 0)
+			return NavigationFile;
+
+		var targets = string.Join("|", privateRepositories.Keys);
+
+		var tempFile = Path.Combine(TemporaryDirectory.FullName, "navigation.filtered.yml");
+		if (_fileSystem.File.Exists(tempFile))
+			return NavigationFile;
+
+		var spacing = -1;
+		var reindenting = -1;
+		var breakingLine = string.Empty;
+		foreach (var l in _fileSystem.File.ReadAllLines(NavigationFile.FullName))
+		{
+			var line = l;
+			if (spacing > -1 && !string.IsNullOrWhiteSpace(line) && !line.StartsWith(new string(' ', spacing), StringComparison.Ordinal))
+			{
+				spacing = -1;
+				reindenting = -1;
+			}
+
+			if (spacing != -1 && Regex.IsMatch(line, $@"^\s{{{spacing}}}\S"))
+			{
+				spacing = -1;
+				reindenting = -1;
+			}
+
+			else if (spacing != -1 && Regex.IsMatch(line, $@"^(\s{{{spacing + 3},}})\S"))
+			{
+				var matches = Regex.Match(line, $@"^(?<spacing>\s{{{spacing}}})(?<remainder>.+)$");
+				line = $"{new string(' ', Math.Max(0, spacing - 4))}{matches.Groups["remainder"].Value}";
+				reindenting = spacing;
+			}
+
+			else if (spacing == -1 && TocPrefixRegex().IsMatch(line))
+			{
+				var matches = Regex.Match(line, $@"^(?<spacing>\s+)-\s?toc:\s?(?:{targets})\:");
+				if (matches.Success)
+				{
+					spacing = matches.Groups["spacing"].Value.Length;
+					breakingLine = line;
+				}
+				if (spacing == 0)
+					spacing = -1;
+			}
+
+			if (spacing == -1 || reindenting > 0)
+				_fileSystem.File.AppendAllLines(tempFile, [line]);
+		}
+		NavigationFile = _fileSystem.FileInfo.New(tempFile);
+		return NavigationFile;
+
+
+	}
+
 	private IFileInfo CreateTemporaryConfigurationFile(string fileName)
 	{
 		using var stream = GetLocalOrEmbedded(fileName);
@@ -90,14 +153,19 @@ private StreamReader GetEmbeddedStream(string fileName)
 
 	private static string GetLocalPath(string file) => Path.Combine(LocalConfigurationDirectory, file);
 	private static string GetAppDataPath(string file) => Path.Combine(AppDataConfigurationDirectory, file);
+	[GeneratedRegex(@"^\s+-?\s?toc:\s?")]
+	private static partial Regex TocPrefixRegex();
 }
 
 public static class ConfigurationFileProviderServiceCollectionExtensions
 {
-	public static IServiceCollection AddConfigurationFileProvider(this IServiceCollection services,
-		Action<IServiceCollection, ConfigurationFileProvider> configure)
+	public static IServiceCollection AddConfigurationFileProvider(
+		this IServiceCollection services,
+		bool skipPrivateRepositories,
+		Action<IServiceCollection, ConfigurationFileProvider> configure
+	)
 	{
-		var provider = new ConfigurationFileProvider(new FileSystem());
+		var provider = new ConfigurationFileProvider(new FileSystem(), skipPrivateRepositories);
 		_ = services.AddSingleton(provider);
 		configure(services, provider);
 		return services;

src/Elastic.Documentation.ServiceDefaults/AppDefaultsExtensions.cs

@@ -2,10 +2,8 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
-using Actions.Core.Extensions;
 using Elastic.Documentation.Configuration;
 using Elastic.Documentation.Configuration.Versions;
-using Elastic.Documentation.Diagnostics;
 using Elastic.Documentation.ServiceDefaults.Logging;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.DependencyInjection.Extensions;
@@ -17,41 +15,32 @@ namespace Elastic.Documentation.ServiceDefaults;
 
 public static class AppDefaultsExtensions
 {
-	public static TBuilder AddAppDefaults<TBuilder>(this TBuilder builder) where TBuilder : IHostApplicationBuilder
+	public static TBuilder AddDocumentationServiceDefaults<TBuilder>(this TBuilder builder) where TBuilder : IHostApplicationBuilder
 	{
 		var args = Array.Empty<string>();
-		return builder.AddAppDefaults(ref args);
+		return builder.AddDocumentationServiceDefaults(ref args);
 	}
-	public static TBuilder AddAppDefaults<TBuilder>(this TBuilder builder, LogLevel defaultLogLevel) where TBuilder : IHostApplicationBuilder
-	{
-		var args = Array.Empty<string>();
-		return builder.AddAppDefaults(ref args, defaultLogLevel);
-	}
-	public static TBuilder AddAppDefaults<TBuilder>(this TBuilder builder, ref string[] args, Action<IServiceCollection, ConfigurationFileProvider> configure) where TBuilder : IHostApplicationBuilder =>
-		builder.AddAppDefaults(ref args, null, configure);
+	public static TBuilder AddDocumentationServiceDefaults<TBuilder>(this TBuilder builder, ref string[] args, Action<IServiceCollection, ConfigurationFileProvider> configure) where TBuilder : IHostApplicationBuilder =>
+		builder.AddDocumentationServiceDefaults(ref args, null, configure);
 
-	public static TBuilder AddAppDefaults<TBuilder>(this TBuilder builder, ref string[] args, LogLevel? defaultLogLevel = null, Action<IServiceCollection, ConfigurationFileProvider>? configure = null) where TBuilder : IHostApplicationBuilder
+	public static TBuilder AddDocumentationServiceDefaults<TBuilder>(this TBuilder builder, ref string[] args, LogLevel? defaultLogLevel = null, Action<IServiceCollection, ConfigurationFileProvider>? configure = null) where TBuilder : IHostApplicationBuilder
 	{
 		var logLevel = defaultLogLevel ?? LogLevel.Information;
-		ProcessCommandLineArguments(ref args, ref logLevel);
+		GlobalCommandLine.Process(ref args, ref logLevel, out var skipPrivateRepositories);
 
 		var services = builder.Services;
 		_ = services
-			.AddGitHubActionsCore()
-			.AddSingleton<DiagnosticsChannel>()
-			.AddSingleton<DiagnosticsCollector>()
-			.AddSingleton<ConfigurationFileProvider>()
-			.AddConfigurationFileProvider((s, p) =>
+			.AddConfigurationFileProvider(skipPrivateRepositories, (s, p) =>
 			{
 				_ = s.AddSingleton(p.CreateVersionConfiguration());
 				configure?.Invoke(s, p);
 			});
-		_ = builder.Services.AddAppLogging(logLevel);
+		_ = builder.Services.AddElasticDocumentationLogging(logLevel);
 
 		return builder.AddServiceDefaults();
 	}
 
-	public static TServiceCollection AddAppLogging<TServiceCollection>(this TServiceCollection services, LogLevel logLevel)
+	public static TServiceCollection AddElasticDocumentationLogging<TServiceCollection>(this TServiceCollection services, LogLevel logLevel)
 		where TServiceCollection : IServiceCollection
 	{
 		services.TryAddEnumerable(ServiceDescriptor.Singleton<ConsoleFormatter, CondensedConsoleFormatter>());
@@ -63,34 +52,4 @@ public static TServiceCollection AddAppLogging<TServiceCollection>(this TService
 		return services;
 	}
 
-	private static void ProcessCommandLineArguments(ref string[] args, ref LogLevel defaultLogLevel)
-	{
-		var newArgs = new List<string>();
-		for (var i = 0; i < args.Length; i++)
-		{
-			if (args[i] == "--log-level")
-			{
-				if (args.Length > i + 1)
-					defaultLogLevel = GetLogLevel(args[i + 1]);
-
-				i++;
-			}
-			else
-				newArgs.Add(args[i]);
-		}
-
-		args = [.. newArgs];
-	}
-
-	private static LogLevel GetLogLevel(string? logLevel) => logLevel switch
-	{
-		"trace" => LogLevel.Trace,
-		"debug" => LogLevel.Debug,
-		"information" => LogLevel.Information,
-		"info" => LogLevel.Information,
-		"warning" => LogLevel.Warning,
-		"error" => LogLevel.Error,
-		"critical" => LogLevel.Critical,
-		_ => LogLevel.Information
-	};
 }

src/Elastic.Documentation/GlobalCommandLine.cs

@@ -0,0 +1,47 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Microsoft.Extensions.Logging;
+
+namespace Elastic.Documentation;
+
+public static class GlobalCommandLine
+{
+	public static void Process(ref string[] args, ref LogLevel defaultLogLevel, out bool skipPrivateRepositories)
+	{
+		skipPrivateRepositories = false;
+		var newArgs = new List<string>();
+		for (var i = 0; i < args.Length; i++)
+		{
+			if (args[i] == "--log-level")
+			{
+				if (args.Length > i + 1)
+					defaultLogLevel = GetLogLevel(args[i + 1]);
+				i++;
+			}
+			else if (args[i] == "--skip-private-repositories")
+			{
+				skipPrivateRepositories = true;
+				i++;
+			}
+			else
+				newArgs.Add(args[i]);
+		}
+
+		args = [.. newArgs];
+	}
+
+	private static LogLevel GetLogLevel(string? logLevel) => logLevel switch
+	{
+		"trace" => LogLevel.Trace,
+		"debug" => LogLevel.Debug,
+		"information" => LogLevel.Information,
+		"info" => LogLevel.Information,
+		"warning" => LogLevel.Warning,
+		"error" => LogLevel.Error,
+		"critical" => LogLevel.Critical,
+		_ => LogLevel.Information
+	};
+
+}

src/api/Elastic.Documentation.Api.Infrastructure/ServicesExtension.cs

@@ -40,9 +40,7 @@ public static class ServicesExtension
 	public static void AddElasticDocsApiUsecases(this IServiceCollection services, string? appEnvironment)
 	{
 		if (AppEnvExtensions.TryParse(appEnvironment, out var parsedEnvironment, true))
-		{
 			AddElasticDocsApiUsecases(services, parsedEnvironment);
-		}
 		else
 		{
 			var logger = GetLogger(services);