Collect all redirects during assemble builds (#1207)

* Collect all redirects during assemble builds

* Fix merge for builder

* Fix tests

---------

Co-authored-by: Felipe Cotti <[email protected]>

Author: Mpdreamz

src/Elastic.Markdown/BuildContext.cs

@@ -30,7 +30,7 @@ public record BuildContext : IDocumentationContext
 
 	public bool Force { get; init; }
 
-	public bool SkipMetadata { get; init; }
+	public bool SkipDocumentationState { get; init; }
 
 	// This property is used to determine if the site should be indexed by search engines
 	public bool AllowIndexing { get; init; }

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -6,6 +6,7 @@
 using System.Reflection;
 using System.Text.Json;
 using Elastic.Documentation.Legacy;
+using Elastic.Documentation.Links;
 using Elastic.Documentation.Serialization;
 using Elastic.Documentation.State;
 using Elastic.Markdown.Exporters;
@@ -27,6 +28,11 @@ public interface IDocumentationFileOutputProvider
 	IFileInfo? OutputFile(DocumentationSet documentationSet, IFileInfo defaultOutputFile, string relativePath);
 }
 
+public record GenerationResult
+{
+	public IReadOnlyDictionary<string, LinkRedirect> Redirects { get; set; } = new Dictionary<string, LinkRedirect>();
+}
+
 public class DocumentationGenerator
 {
 	private readonly IDocumentationFileOutputProvider? _documentationFileOutputProvider;
@@ -87,14 +93,23 @@ public async Task ResolveDirectoryTree(Cancel ctx)
 		_logger.LogInformation("Resolved tree");
 	}
 
-	public async Task GenerateAll(Cancel ctx)
+	public async Task<GenerationResult> GenerateAll(Cancel ctx)
 	{
-		var generationState = GetPreviousGenerationState();
-		if (!Context.SkipMetadata && (Context.Force || generationState == null))
+		var result = new GenerationResult();
+
+		HashSet<string> offendingFiles = [];
+		var outputSeenChanges = DateTimeOffset.MinValue;
+		if (Context.SkipDocumentationState)
 			DocumentationSet.ClearOutputDirectory();
+		else
+		{
+			var generationState = GetPreviousGenerationState();
+			if (Context.Force || generationState == null)
+				DocumentationSet.ClearOutputDirectory();
 
-		if (CompilationNotNeeded(generationState, out var offendingFiles, out var outputSeenChanges))
-			return;
+			if (CompilationNotNeeded(generationState, out offendingFiles, out outputSeenChanges))
+				return result;
+		}
 
 		_logger.LogInformation($"Fetching external links");
 		_ = await Resolver.FetchLinks(ctx);
@@ -107,14 +122,20 @@ public async Task GenerateAll(Cancel ctx)
 
 		await ExtractEmbeddedStaticResources(ctx);
 
-		if (Context.SkipMetadata)
-			return;
-
-		_logger.LogInformation($"Generating documentation compilation state");
-		await GenerateDocumentationState(ctx);
+		if (!Context.SkipDocumentationState)
+		{
+			_logger.LogInformation($"Generating documentation compilation state");
+			await GenerateDocumentationState(ctx);
+		}
 
 		_logger.LogInformation($"Generating links.json");
-		await GenerateLinkReference(ctx);
+		var linkReference = await GenerateLinkReference(ctx);
+
+		// ReSharper disable once WithExpressionModifiesAllMembers
+		return result with
+		{
+			Redirects = linkReference.Redirects ?? []
+		};
 	}
 
 	private async Task ProcessDocumentationFiles(HashSet<string> offendingFiles, DateTimeOffset outputSeenChanges, Cancel ctx)
@@ -254,13 +275,13 @@ private bool CompilationNotNeeded(GenerationState? generationState, out HashSet<
 		return false;
 	}
 
-	private async Task GenerateLinkReference(Cancel ctx)
+	private async Task<LinkReference> GenerateLinkReference(Cancel ctx)
 	{
 		var file = DocumentationSet.LinkReferenceFile;
 		var state = DocumentationSet.CreateLinkReference();
-
 		var bytes = JsonSerializer.SerializeToUtf8Bytes(state, SourceGenerationContext.Default.LinkReference);
 		await DocumentationSet.OutputDirectory.FileSystem.File.WriteAllBytesAsync(file.FullName, bytes, ctx);
+		return state;
 	}
 
 	private async Task GenerateDocumentationState(Cancel ctx)

src/tooling/docs-assembler/AssembleContext.cs

@@ -72,7 +72,8 @@ public AssembleContext(
 		Environment = env;
 
 		var contentSource = Environment.ContentSource.ToStringFast(true);
-		CheckoutDirectory = ReadFileSystem.DirectoryInfo.New(checkoutDirectory ?? Path.Combine(".artifacts", "checkouts", contentSource));
+		var defaultCheckoutDirectory = Path.Combine(Paths.WorkingDirectoryRoot.FullName, ".artifacts", "checkouts", contentSource);
+		CheckoutDirectory = ReadFileSystem.DirectoryInfo.New(checkoutDirectory ?? defaultCheckoutDirectory);
 		OutputDirectory = ReadFileSystem.DirectoryInfo.New(output ?? Path.Combine(".artifacts", "assembly"));
 
 

src/tooling/docs-assembler/Building/AssemblerBuilder.cs

@@ -5,7 +5,9 @@
 using System.Collections.Frozen;
 using Documentation.Assembler.Navigation;
 using Elastic.Documentation.Legacy;
+using Elastic.Documentation.Links;
 using Elastic.Markdown;
+using Elastic.Markdown.Links.CrossLinks;
 using Microsoft.Extensions.Logging;
 
 namespace Documentation.Assembler.Building;
@@ -29,6 +31,8 @@ public async Task BuildAllAsync(FrozenDictionary<string, AssemblerDocumentationS
 			context.OutputDirectory.Delete(true);
 		context.OutputDirectory.Create();
 
+		var redirects = new Dictionary<string, string>();
+
 		foreach (var (_, set) in assembleSets)
 		{
 			var checkout = set.Checkout;
@@ -40,7 +44,8 @@ public async Task BuildAllAsync(FrozenDictionary<string, AssemblerDocumentationS
 
 			try
 			{
-				await BuildAsync(set, ctx);
+				var result = await BuildAsync(set, ctx);
+				CollectRedirects(redirects, result.Redirects, checkout.Repository.Name, set.DocumentationSet.LinkResolver);
 			}
 			catch (Exception e) when (e.Message.Contains("Can not locate docset.yml file in"))
 			{
@@ -56,7 +61,36 @@ public async Task BuildAllAsync(FrozenDictionary<string, AssemblerDocumentationS
 		await context.Collector.StopAsync(ctx);
 	}
 
-	private async Task BuildAsync(AssemblerDocumentationSet set, Cancel ctx)
+	private static void CollectRedirects(
+		Dictionary<string, string> allRedirects,
+		IReadOnlyDictionary<string, LinkRedirect> redirects,
+		string repository,
+		ICrossLinkResolver linkResolver
+	)
+	{
+		if (redirects.Count == 0)
+			return;
+
+		foreach (var (k, v) in redirects)
+		{
+			if (v.To is { } to)
+				allRedirects[Resolve(k)] = Resolve(to);
+			else if (v.Many is { } many)
+			{
+				var target = many.FirstOrDefault(l => l.To is not null);
+				if (target?.To is { } t)
+					allRedirects[Resolve(k)] = Resolve(t);
+			}
+		}
+		string Resolve(string relativeMarkdownPath)
+		{
+			var uri = linkResolver.UriResolver.Resolve(new Uri($"{repository}://{relativeMarkdownPath}"),
+				PublishEnvironmentUriResolver.MarkdownPathToUrlPath(relativeMarkdownPath));
+			return uri.AbsolutePath;
+		}
+	}
+
+	private async Task<GenerationResult> BuildAsync(AssemblerDocumentationSet set, Cancel ctx)
 	{
 		var generator = new DocumentationGenerator(
 			set.DocumentationSet,
@@ -65,7 +99,7 @@ private async Task BuildAsync(AssemblerDocumentationSet set, Cancel ctx)
 			legacyUrlMapper: LegacyUrlMapper,
 			positionalNavigation: navigation
 		);
-		await generator.GenerateAll(ctx);
+		return await generator.GenerateAll(ctx);
 	}
 
 }

src/tooling/docs-assembler/Building/PublishEnvironmentUriResolver.cs

@@ -41,11 +41,6 @@ public PublishEnvironmentUriResolver(FrozenDictionary<Uri, TocTopLevelMapping> t
 
 	public Uri Resolve(Uri crossLinkUri, string path)
 	{
-		if (crossLinkUri.Scheme == "detection-rules")
-		{
-
-		}
-
 		var subPath = GetSubPathPrefix(crossLinkUri, ref path);
 
 		var fullPath = (PublishEnvironment.PathPrefix, subPath) switch

src/tooling/docs-assembler/Cli/RepositoryCommands.cs

@@ -164,7 +164,7 @@ await Parallel.ForEachAsync(repositories,
 					);
 					var set = new DocumentationSet(context, logger);
 					var generator = new DocumentationGenerator(set, logger, null, null, new NoopDocumentationFileExporter());
-					await generator.GenerateAll(c);
+					_ = await generator.GenerateAll(c);
 
 					IAmazonS3 s3Client = new AmazonS3Client();
 					const string bucketName = "elastic-docs-link-index";

src/tooling/docs-assembler/Navigation/AssemblerDocumentationSet.cs

@@ -69,7 +69,7 @@ public AssemblerDocumentationSet(
 				CookiesWin = env.GoogleTagManager.CookiesWin
 			},
 			CanonicalBaseUrl = new Uri("https://www.elastic.co"), // Always use the production URL. In case a page is leaked to a search engine, it should point to the production site.
-			SkipMetadata = true,
+			SkipDocumentationState = true,
 		};
 		BuildContext = buildContext;
 

src/tooling/docs-builder/Cli/Commands.cs

@@ -153,7 +153,7 @@ public async Task<int> Generate(
 		var exporter = metadataOnly.HasValue && metadataOnly.Value ? new NoopDocumentationFileExporter() : null;
 
 		var generator = new DocumentationGenerator(set, logger, null, null, exporter);
-		await generator.GenerateAll(ctx);
+		_ = await generator.GenerateAll(ctx);
 
 		if (runningOnCi)
 			await githubActionsService.SetOutputAsync("landing-page-path", set.MarkdownFiles.First().Value.Url);

tests/authoring/Framework/TestValues.fs

@@ -99,7 +99,7 @@ and MarkdownTestContext =
     member this.Bootstrap () = backgroundTask {
         let! ctx = Async.CancellationToken
         let _ = this.Collector.StartAsync(ctx)
-        do! this.Generator.GenerateAll(ctx)
+        let! _ = this.Generator.GenerateAll(ctx)
         do! this.Collector.StopAsync(ctx)
 
         let results =