Merge docs-assembler commands into docs-builder

Directory.Packages.props

@@ -97,4 +97,4 @@
     </PackageVersion>
     <PackageVersion Include="xunit.v3" Version="2.0.2" />
   </ItemGroup>
-</Project>
+</Project>

aspire/AppHost.cs

@@ -40,12 +40,12 @@ async Task BuildAspireHost(bool startElasticsearch, bool assumeCloned, bool skip
 	var elasticsearchUrl = builder.AddParameter("DocumentationElasticUrl", secret: true);
 	var elasticsearchApiKey = builder.AddParameter("DocumentationElasticApiKey", secret: true);
 
-	var cloneAll = builder.AddProject<Projects.docs_assembler>(AssemblerClone);
+	var cloneAll = builder.AddProject<Projects.docs_builder>(AssemblerClone);
 	string[] cloneArgs = assumeCloned ? ["--assume-cloned"] : [];
-	cloneAll = cloneAll.WithArgs(["repo", "clone-all", .. globalArguments, .. cloneArgs]);
+	cloneAll = cloneAll.WithArgs(["assembler", "clone", .. globalArguments, .. cloneArgs]);
 
-	var buildAll = builder.AddProject<Projects.docs_assembler>(AssemblerBuild)
-		.WithArgs(["repo", "build-all", .. globalArguments])
+	var buildAll = builder.AddProject<Projects.docs_builder>(AssemblerBuild)
+		.WithArgs(["assembler", "build", .. globalArguments])
 		.WaitForCompletion(cloneAll)
 		.WithParentRelationship(cloneAll);
 
@@ -74,8 +74,8 @@ async Task BuildAspireHost(bool startElasticsearch, bool assumeCloned, bool skip
 			.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchUrl)
 			.WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey);
 
-	var indexElasticsearch = builder.AddProject<Projects.docs_assembler>(ElasticsearchIndexerPlain)
-		.WithArgs(["repo", "build-all", "--exporters", "elasticsearch", .. globalArguments])
+	var indexElasticsearch = builder.AddProject<Projects.docs_builder>(ElasticsearchIndexerPlain)
+		.WithArgs(["assembler", "build", "--exporters", "elasticsearch", .. globalArguments])
 		.WithExplicitStart()
 		.WaitForCompletion(cloneAll);
 	indexElasticsearch = startElasticsearch
@@ -91,8 +91,8 @@ async Task BuildAspireHost(bool startElasticsearch, bool assumeCloned, bool skip
 			.WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey)
 			.WithParentRelationship(elasticsearchRemote);
 
-	var indexElasticsearchSemantic = builder.AddProject<Projects.docs_assembler>(ElasticsearchIndexerSemantic)
-		.WithArgs(["repo", "build-all", "--exporters", "semantic", .. globalArguments])
+	var indexElasticsearchSemantic = builder.AddProject<Projects.docs_builder>(ElasticsearchIndexerSemantic)
+		.WithArgs(["assembler", "build", "--exporters", "semantic", .. globalArguments])
 		.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http"))
 		.WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter)
 		.WithExplicitStart()
@@ -114,7 +114,7 @@ async Task BuildAspireHost(bool startElasticsearch, bool assumeCloned, bool skip
 		.WithEnvironment("LLM_GATEWAY_FUNCTION_URL", llmUrl)
 		.WithEnvironment("LLM_GATEWAY_SERVICE_ACCOUNT_KEY_PATH", llmServiceAccountPath)
 		.WithHttpEndpoint(port: 4000, isProxied: false)
-		.WithArgs(["serve-static", .. globalArguments])
+		.WithArgs(["assembler", "serve", .. globalArguments])
 		.WithHttpHealthCheck("/", 200)
 		.WaitForCompletion(buildAll)
 		.WithParentRelationship(cloneAll);

docs-builder.sln

@@ -147,6 +147,16 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "diff-validate", "diff-valid
 		actions\diff-validate\action.yml = actions\diff-validate\action.yml
 	EndProjectSection
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "services", "services", "{7AACA67B-3C56-4C7C-9891-558589FC52DB}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Documentation.Assembler", "src\services\Elastic.Documentation.Assembler\Elastic.Documentation.Assembler.csproj", "{094433A4-504F-4E12-959F-CCB1965C1C9A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Documentation.Services", "src\services\Elastic.Documentation.Services\Elastic.Documentation.Services.csproj", "{E6EA955D-D0A7-4749-9586-0F7256EF5C5E}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Documentation.Links", "src\Elastic.Documentation.Links\Elastic.Documentation.Links.csproj", "{153FC4AD-F5B0-4100-990E-0987C86DBF01}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Documentation.Isolated", "src\services\Elastic.Documentation.Isolated\Elastic.Documentation.Isolated.csproj", "{AABD3EF7-8C86-4981-B1D2-B1F786F33069}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -256,6 +266,22 @@ Global
 		{C6A121C5-DEB1-4FCE-9140-AF144EA98EEE}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C6A121C5-DEB1-4FCE-9140-AF144EA98EEE}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C6A121C5-DEB1-4FCE-9140-AF144EA98EEE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{094433A4-504F-4E12-959F-CCB1965C1C9A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{094433A4-504F-4E12-959F-CCB1965C1C9A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{094433A4-504F-4E12-959F-CCB1965C1C9A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{094433A4-504F-4E12-959F-CCB1965C1C9A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E6EA955D-D0A7-4749-9586-0F7256EF5C5E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E6EA955D-D0A7-4749-9586-0F7256EF5C5E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E6EA955D-D0A7-4749-9586-0F7256EF5C5E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E6EA955D-D0A7-4749-9586-0F7256EF5C5E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{153FC4AD-F5B0-4100-990E-0987C86DBF01}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{153FC4AD-F5B0-4100-990E-0987C86DBF01}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{153FC4AD-F5B0-4100-990E-0987C86DBF01}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{153FC4AD-F5B0-4100-990E-0987C86DBF01}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AABD3EF7-8C86-4981-B1D2-B1F786F33069}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AABD3EF7-8C86-4981-B1D2-B1F786F33069}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AABD3EF7-8C86-4981-B1D2-B1F786F33069}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AABD3EF7-8C86-4981-B1D2-B1F786F33069}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{4D198E25-C211-41DC-9E84-B15E89BD7048} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
@@ -293,5 +319,10 @@ Global
 		{AE3FC78E-167F-4B6E-88EC-84743EB748B7} = {B042CC78-5060-4091-B95A-79C71BA3908A}
 		{C6A121C5-DEB1-4FCE-9140-AF144EA98EEE} = {B042CC78-5060-4091-B95A-79C71BA3908A}
 		{E7C7A02B-9AB4-455A-9A64-3D326ED1A95A} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
+		{7AACA67B-3C56-4C7C-9891-558589FC52DB} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
+		{094433A4-504F-4E12-959F-CCB1965C1C9A} = {7AACA67B-3C56-4C7C-9891-558589FC52DB}
+		{E6EA955D-D0A7-4749-9586-0F7256EF5C5E} = {7AACA67B-3C56-4C7C-9891-558589FC52DB}
+		{153FC4AD-F5B0-4100-990E-0987C86DBF01} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
+		{AABD3EF7-8C86-4981-B1D2-B1F786F33069} = {7AACA67B-3C56-4C7C-9891-558589FC52DB}
 	EndGlobalSection
 EndGlobal

src/tooling/docs-assembler/Navigation/GlobalNavigationFile.cs → src/Elastic.Documentation.Configuration/Navigation/GlobalNavigationFile.cs

@@ -2,43 +2,75 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System.Collections.Frozen;
 using System.Collections.Immutable;
 using System.IO.Abstractions;
-using Elastic.Documentation;
-using Elastic.Documentation.Configuration;
 using Elastic.Documentation.Configuration.Assembler;
+using Elastic.Documentation.Configuration.Builder;
 using Elastic.Documentation.Configuration.TableOfContents;
+using Elastic.Documentation.Diagnostics;
 using Elastic.Documentation.Navigation;
 using YamlDotNet.RepresentationModel;
 
-namespace Documentation.Assembler.Navigation;
+namespace Elastic.Documentation.Configuration.Navigation;
+
+public record NavigationTocMapping
+{
+	public required Uri Source { get; init; }
+	public required string SourcePathPrefix { get; init; }
+	public required Uri TopLevelSource { get; init; }
+	public required Uri ParentSource { get; init; }
+}
+
+public record TocConfigurationMapping
+{
+	public required NavigationTocMapping TopLevel { get; init; }
+	public required ConfigurationFile RepositoryConfigurationFile { get; init; }
+	public required TableOfContentsConfiguration TableOfContentsConfiguration { get; init; }
+}
 
 public record GlobalNavigationFile : ITableOfContentsScope
 {
-	private readonly AssembleContext _context;
-	private readonly AssembleSources _assembleSources;
+	//private readonly AssembleContext _context;
+	private readonly IDiagnosticsCollector _collector;
+	private readonly ConfigurationFileProvider _configurationFileProvider;
+	private readonly AssemblyConfiguration _configuration;
+
+	private readonly FrozenDictionary<Uri, TocConfigurationMapping> _tocConfigurationMappings;
+	//private readonly AssembleSources _assembleSources;
 
 	public IReadOnlyCollection<TocReference> TableOfContents { get; }
 	public IReadOnlyCollection<TocReference> Phantoms { get; }
 
 	public IDirectoryInfo ScopeDirectory { get; }
 
-	public GlobalNavigationFile(AssembleContext context, AssembleSources assembleSources)
+	public GlobalNavigationFile(
+		IDiagnosticsCollector collector,
+		ConfigurationFileProvider configurationFileProvider,
+		AssemblyConfiguration configuration,
+		FrozenDictionary<Uri, TocConfigurationMapping> tocConfigurationMappings
+	)
 	{
-		_context = context;
-		_assembleSources = assembleSources;
-		NavigationFile = context.ConfigurationFileProvider.CreateNavigationFile(context.Configuration);
+		//_context = context;
+		_collector = collector;
+		_configurationFileProvider = configurationFileProvider;
+		_configuration = configuration;
+		_tocConfigurationMappings = tocConfigurationMappings;
+		NavigationFile = configurationFileProvider.CreateNavigationFile(configuration);
 		TableOfContents = Deserialize("toc");
 		Phantoms = Deserialize("phantoms");
 		ScopeDirectory = NavigationFile.Directory!;
 	}
 
 	private IFileInfo NavigationFile { get; }
 
-	public static bool ValidatePathPrefixes(AssembleContext context)
+	public static bool ValidatePathPrefixes(
+		IDiagnosticsCollector collector,
+		ConfigurationFileProvider configurationFileProvider,
+		AssemblyConfiguration configuration
+	)
 	{
-		var fileProvider = context.ConfigurationFileProvider;
-		var sourcePathPrefixes = GetAllPathPrefixes(context);
+		var sourcePathPrefixes = GetAllPathPrefixes(collector, configurationFileProvider, configuration);
 		var pathPrefixSet = new HashSet<string>();
 		var valid = true;
 		foreach (var pathPrefix in sourcePathPrefixes)
@@ -47,23 +79,36 @@ public static bool ValidatePathPrefixes(AssembleContext context)
 			if (pathPrefixSet.Add(prefix))
 				continue;
 			var duplicateOf = sourcePathPrefixes.First(p => p.Host == pathPrefix.Host && p.AbsolutePath == pathPrefix.AbsolutePath);
-			context.Collector.EmitError(fileProvider.NavigationFile, $"Duplicate path prefix: {pathPrefix} duplicate: {duplicateOf}");
+			collector.EmitError(configurationFileProvider.NavigationFile, $"Duplicate path prefix: {pathPrefix} duplicate: {duplicateOf}");
 			valid = false;
 		}
 		return valid;
 	}
 
 
-	public static ImmutableHashSet<Uri> GetAllPathPrefixes(AssembleContext context) =>
-		GetSourceUris("toc", context);
-
-	public static ImmutableHashSet<Uri> GetPhantomPrefixes(AssembleContext context) =>
-		GetSourceUris("phantoms", context);
-
-	private static ImmutableHashSet<Uri> GetSourceUris(string key, AssembleContext context)
+	public static ImmutableHashSet<Uri> GetAllPathPrefixes(
+		IDiagnosticsCollector collector,
+		ConfigurationFileProvider configurationFileProvider,
+		AssemblyConfiguration configuration
+	) =>
+		GetSourceUris("toc", collector, configurationFileProvider, configuration);
+
+	public static ImmutableHashSet<Uri> GetPhantomPrefixes(
+		IDiagnosticsCollector collector,
+		ConfigurationFileProvider configurationFileProvider,
+		AssemblyConfiguration configuration
+	) =>
+		GetSourceUris("phantoms", collector, configurationFileProvider, configuration);
+
+	private static ImmutableHashSet<Uri> GetSourceUris(
+		string key,
+		IDiagnosticsCollector collector,
+		ConfigurationFileProvider configurationFileProvider,
+		AssemblyConfiguration configuration
+	)
 	{
-		var navigationFile = context.ConfigurationFileProvider.CreateNavigationFile(context.Configuration);
-		var reader = new YamlStreamReader(navigationFile, context.Collector);
+		var navigationFile = configurationFileProvider.CreateNavigationFile(configuration);
+		var reader = new YamlStreamReader(navigationFile, collector);
 		var set = new HashSet<Uri>();
 		foreach (var entry in reader.Read())
 		{
@@ -145,15 +190,15 @@ static void ReadPathPrefixes(YamlStreamReader reader, KeyValuePair<YamlNode, Yam
 		}
 	}
 	public void EmitWarning(string message) =>
-		_context.Collector.EmitWarning(NavigationFile, message);
+		_collector.EmitWarning(NavigationFile, message);
 
 	public void EmitError(string message) =>
-		_context.Collector.EmitError(NavigationFile, message);
+		_collector.EmitError(NavigationFile, message);
 
 	private IReadOnlyCollection<TocReference> Deserialize(string key)
 	{
-		var navigationFile = _context.ConfigurationFileProvider.CreateNavigationFile(_context.Configuration);
-		var reader = new YamlStreamReader(navigationFile, _context.Collector);
+		var navigationFile = _configurationFileProvider.CreateNavigationFile(_configuration);
+		var reader = new YamlStreamReader(navigationFile, _collector);
 		try
 		{
 			foreach (var entry in reader.Read())
@@ -232,7 +277,7 @@ private IReadOnlyCollection<TocReference> ReadChildren(string key, YamlStreamRea
 			return null;
 
 
-		if (!_assembleSources.TocConfigurationMapping.TryGetValue(sourceUri, out var mapping))
+		if (!_tocConfigurationMappings.TryGetValue(sourceUri, out var mapping))
 		{
 			reader.EmitError($"Toc entry '{sourceUri}' is could not be located", tocEntry);
 			return null;
@@ -270,7 +315,7 @@ private IReadOnlyCollection<TocReference> ReadChildren(string key, YamlStreamRea
 		if (source is null)
 			return pathPrefix;
 
-		source = source.EndsWith("://") ? source : source.TrimEnd('/') + "/";
+		source = source.EndsWith("://", StringComparison.OrdinalIgnoreCase) ? source : source.TrimEnd('/') + "/";
 		if (!Uri.TryCreate(source, UriKind.Absolute, out sourceUri))
 		{
 			reader.EmitError($"Source toc entry is not a valid uri: {source}", tocEntry);

src/Elastic.Documentation.Configuration/Paths.cs

@@ -16,9 +16,17 @@ public static class Paths
 	private static DirectoryInfo DetermineWorkingDirectoryRoot()
 	{
 		var directory = new DirectoryInfo(Directory.GetCurrentDirectory());
-		while (directory != null &&
-			   (directory.GetFiles("*.sln").Length == 0 || directory.GetDirectories(".git").Length == 0))
+		while (directory != null)
+		{
+			if (directory.GetFiles("*.sln").Length > 0)
+				break;
+			if (directory.GetDirectories(".git").Length > 0)
+				break;
+			// support for git worktrees
+			if (directory.GetFiles(".git").Length > 0)
+				break;
 			directory = directory.Parent;
+		}
 		return directory ?? new DirectoryInfo(Directory.GetCurrentDirectory());
 	}
 
@@ -27,12 +35,20 @@ private static DirectoryInfo DetermineWorkingDirectoryRoot()
 		IDirectoryInfo? sourceRoot = null;
 		var directory = sourceDirectory;
 		while (directory != null && directory.GetDirectories(".git").Length == 0)
+		{
+			if (directory.GetDirectories(".git").Length > 0)
+				break;
+			// support for git worktrees
+			if (directory.GetFiles(".git").Length > 0)
+				break;
+
 			directory = directory.Parent;
+		}
 		sourceRoot ??= directory;
 		return sourceRoot;
 	}
 
-	/// Used in debug to locate static folder so we can change js/css files while the server is running
+	/// Used in debug to locate static folder, so we can change js/css files while the server is running
 	public static DirectoryInfo? GetSolutionDirectory()
 	{
 		var directory = new DirectoryInfo(Directory.GetCurrentDirectory());

src/Elastic.Documentation.LegacyDocs/Elastic.Documentation.LegacyDocs.csproj

@@ -14,6 +14,7 @@
 
     <ItemGroup>
       <ProjectReference Include="..\Elastic.Documentation.Configuration\Elastic.Documentation.Configuration.csproj" />
+      <ProjectReference Include="..\services\Elastic.Documentation.Services\Elastic.Documentation.Services.csproj" />
     </ItemGroup>
 
   <ItemGroup>

src/Elastic.Documentation.LegacyDocs/LegacyPageChecker.cs → src/Elastic.Documentation.LegacyDocs/LegacyPageService.cs

@@ -3,40 +3,47 @@
 // See the LICENSE file in the project root for more information
 
 using Elastic.Documentation.Configuration;
+using Elastic.Documentation.Services;
+using Microsoft.Extensions.Logging;
 
 namespace Elastic.Documentation.LegacyDocs;
 
-public class LegacyPageChecker
+public class LegacyPageService(ILoggerFactory logFactory) : IService
 {
+	private readonly ILogger _logger = logFactory.CreateLogger<LegacyPageService>();
 	private BloomFilter? _bloomFilter;
 	private const string RootNamespace = "Elastic.Documentation.LegacyDocs";
 	private const string FileName = "legacy-pages.bloom.bin";
 	private const string ResourceName = $"{RootNamespace}.{FileName}";
 	private readonly string _bloomFilterBinaryPath = Path.Combine(Paths.WorkingDirectoryRoot.FullName, "src", RootNamespace, FileName);
 
-
-	public bool PathExists(string path)
+	public bool PathExists(string path, bool logResult = false)
 	{
 		_bloomFilter ??= LoadBloomFilter();
-		return _bloomFilter.Check(path);
+		var exists = _bloomFilter.Check(path);
+		if (logResult)
+			_logger.LogInformation("Path {Path} {Exists} in bloom filter", path, exists ? "exists" : "not exists");
+		return exists;
 	}
 
 	private static BloomFilter LoadBloomFilter()
 	{
-		var assembly = typeof(LegacyPageChecker).Assembly;
+		var assembly = typeof(LegacyPageService).Assembly;
 		using var stream = assembly.GetManifestResourceStream(ResourceName) ?? throw new FileNotFoundException(
 			$"Embedded resource '{ResourceName}' not found in assembly '{assembly.FullName}'. " +
 			"Ensure the Build Action for 'legacy-pages.bloom.bin' is 'Embedded Resource' and the path/name is correct.");
 		return BloomFilter.Load(stream);
 	}
 
-	public void GenerateBloomFilterBinary(IPagesProvider pagesProvider)
+	public bool GenerateBloomFilterBinary(IPagesProvider pagesProvider)
 	{
 		var pages = pagesProvider.GetPages();
 		var enumerable = pages as string[] ?? pages.ToArray();
 		var paths = enumerable.ToHashSet();
 		var bloomFilter = BloomFilter.FromCollection(enumerable, 0.001);
 		Console.WriteLine(paths.Count);
 		bloomFilter.Save(_bloomFilterBinaryPath);
+		_logger.LogInformation("Bloom filter generated to {Path}", _bloomFilterBinaryPath);
+		return true;
 	}
 }