Upload links.json to s3

src/docs-builder/Cli/Commands.cs

@@ -4,16 +4,16 @@
 using System.IO.Abstractions;
 using Actions.Core.Services;
 using ConsoleAppFramework;
-using Documentation.Builder.Diagnostics;
 using Documentation.Builder.Diagnostics.Console;
 using Documentation.Builder.Http;
+using Documentation.Builder.LinkIndex;
 using Elastic.Markdown;
 using Elastic.Markdown.IO;
 using Microsoft.Extensions.Logging;
 
 namespace Documentation.Builder.Cli;
 
-internal class Commands(ILoggerFactory logger, ICoreService githubActionsService)
+internal class Commands(ILoggerFactory logger, ICoreService githubActionsService, ILinkIndex linkIndex)
 {
 	/// <summary>
 	///	Continuously serve a documentation folder at http://localhost:5000.
@@ -29,7 +29,6 @@ public async Task Serve(string? path = null, Cancel ctx = default)
 		var host = new DocumentationWebHost(path, logger, new FileSystem());
 		await host.RunAsync(ctx);
 		await host.StopAsync(ctx);
-
 	}
 
 	/// <summary>
@@ -39,6 +38,7 @@ public async Task Serve(string? path = null, Cancel ctx = default)
 	/// <param name="output"> -o, Defaults to `.artifacts/html` </param>
 	/// <param name="pathPrefix"> Specifies the path prefix for urls </param>
 	/// <param name="force"> Force a full rebuild of the destination folder</param>
+	/// <param name="uploadToLinkIndex"> Upload the links.json file to the link index</param>
 	/// <param name="ctx"></param>
 	[Command("generate")]
 	[ConsoleAppFilter<StopwatchFilter>]
@@ -48,6 +48,7 @@ public async Task<int> Generate(
 		string? output = null,
 		string? pathPrefix = null,
 		bool? force = null,
+		bool uploadToLinkIndex = false,
 		Cancel ctx = default
 	)
 	{
@@ -57,11 +58,13 @@ public async Task<int> Generate(
 		{
 			UrlPathPrefix = pathPrefix,
 			Force = force ?? false,
-			Collector = new ConsoleDiagnosticsCollector(logger, githubActionsService)
+			Collector = new ConsoleDiagnosticsCollector(logger, githubActionsService),
 		};
 		var set = new DocumentationSet(context);
 		var generator = new DocumentationGenerator(set, logger);
 		await generator.GenerateAll(ctx);
+		var linksJsonPath = set.LinkReferenceFile.FullName;
+		await linkIndex.UploadFileAsync(linksJsonPath, uploadToLinkIndex);
 		return context.Collector.Errors + context.Collector.Warnings;
 	}
 
@@ -72,6 +75,7 @@ public async Task<int> Generate(
 	/// <param name="output"> -o, Defaults to `.artifacts/html` </param>
 	/// <param name="pathPrefix"> Specifies the path prefix for urls </param>
 	/// <param name="force"> Force a full rebuild of the destination folder</param>
+	/// <param name="uploadToLinkIndex"> Upload the links.json file to the link index</param>
 	/// <param name="ctx"></param>
 	[Command("")]
 	[ConsoleAppFilter<StopwatchFilter>]
@@ -81,7 +85,8 @@ public async Task<int> GenerateDefault(
 		string? output = null,
 		string? pathPrefix = null,
 		bool? force = null,
+		bool uploadToLinkIndex = false,
 		Cancel ctx = default
 	) =>
-		await Generate(path, output, pathPrefix, force, ctx);
+		await Generate(path, output, pathPrefix, force, uploadToLinkIndex, ctx);
 }

src/docs-builder/LinkIndex/ILinkIndex.cs

@@ -0,0 +1,9 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+namespace Documentation.Builder.LinkIndex;
+
+public interface ILinkIndex
+{
+	Task UploadFileAsync(string filePath, bool shouldUpload);
+}

src/docs-builder/LinkIndex/S3LinkIndex.cs

@@ -0,0 +1,61 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+using Amazon.S3;
+using Amazon.S3.Transfer;
+using Microsoft.Extensions.Logging;
+
+namespace Documentation.Builder.LinkIndex;
+
+public class S3LinkIndex : ILinkIndex
+{
+	private readonly string _bucketName;
+	private readonly ILogger<S3LinkIndex> _logger;
+
+	public S3LinkIndex(string bucketName, ILoggerFactory loggerFactory)
+	{
+		_bucketName = bucketName;
+		_logger = loggerFactory.CreateLogger<S3LinkIndex>();
+	}
+
+	public async Task UploadFileAsync(string filePath, bool shouldUpload)
+	{
+		if (!shouldUpload)
+		{
+			_logger.LogInformation("Not uploading link index: skip flag is set");
+			return;
+		}
+
+		var githubRef = Environment.GetEnvironmentVariable("GITHUB_REF");
+		if (string.IsNullOrEmpty(githubRef) || githubRef != "refs/heads/main")
+		{
+			throw new InvalidOperationException($"Cannot upload link index: GITHUB_REF '{githubRef}' is not main branch");
+		}
+
+		var s3DestinationPath = DeriveDestinationPath();
+
+		_logger.LogInformation("Uploading link index {FilePath} to S3://{Bucket}/{DestinationPath}", filePath, _bucketName, s3DestinationPath);
+		using var client = new AmazonS3Client();
+		var fileTransferUtility = new TransferUtility(client);
+		try
+		{
+			await fileTransferUtility.UploadAsync(filePath, _bucketName, s3DestinationPath);
+			_logger.LogInformation("Successfully uploaded link reference {FilePath} to S3://{Bucket}/{DestinationPath}",
+				filePath, _bucketName, s3DestinationPath);
+		}
+		catch (Exception e)
+		{
+			_logger.LogError(e, "Failed to upload link index {FilePath} to S3://{Bucket}/{DestinationPath}",
+				filePath, _bucketName, s3DestinationPath);
+			throw;
+		}
+	}
+
+	private static string DeriveDestinationPath()
+	{
+		var repositoryName = Environment.GetEnvironmentVariable("GITHUB_REPOSITORY")?.Split('/').Last();
+		if (string.IsNullOrEmpty(repositoryName))
+			throw new InvalidOperationException("Cannot upload link index: GITHUB_REPOSITORY environment variable is not set or invalid");
+		return $"{repositoryName}.json";
+	}
+}

src/docs-builder/Program.cs

@@ -6,6 +6,7 @@
 using ConsoleAppFramework;
 using Documentation.Builder;
 using Documentation.Builder.Cli;
+using Documentation.Builder.LinkIndex;
 using Elastic.Markdown.Diagnostics;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
@@ -26,7 +27,8 @@
 });
 services.AddSingleton<DiagnosticsChannel>();
 services.AddSingleton<DiagnosticsCollector>();
-
+services.AddSingleton<S3LinkIndex>(sp => new S3LinkIndex("elastic-docs-link-index", sp.GetRequiredService<ILoggerFactory>()));
+services.AddSingleton<ILinkIndex>(sp => sp.GetRequiredService<S3LinkIndex>());
 
 await using var serviceProvider = services.BuildServiceProvider();
 var logger = serviceProvider.GetRequiredService<ILogger<Program>>();

src/docs-builder/docs-builder.csproj

@@ -19,6 +19,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="AWSSDK.S3" Version="3.7.411.5" />
     <PackageReference Include="ConsoleAppFramework" Version="5.3.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>