Upload links.json to s3

Author: reakaleek

src/docs-builder/Cli/Commands.cs

@@ -4,16 +4,16 @@
 using System.IO.Abstractions;
 using Actions.Core.Services;
 using ConsoleAppFramework;
-using Documentation.Builder.Diagnostics;
 using Documentation.Builder.Diagnostics.Console;
 using Documentation.Builder.Http;
 using Elastic.Markdown;
 using Elastic.Markdown.IO;
 using Microsoft.Extensions.Logging;
+using Documentation.Builder.LinkIndex;
 
 namespace Documentation.Builder.Cli;
 
-internal class Commands(ILoggerFactory logger, ICoreService githubActionsService)
+internal class Commands(ILoggerFactory logger, ICoreService githubActionsService, ILinkIndex linkIndex)
 {
 	/// <summary>
 	///	Continuously serve a documentation folder at http://localhost:5000.
@@ -29,7 +29,6 @@ public async Task Serve(string? path = null, Cancel ctx = default)
 		var host = new DocumentationWebHost(path, logger, new FileSystem());
 		await host.RunAsync(ctx);
 		await host.StopAsync(ctx);
-
 	}
 
 	/// <summary>
@@ -39,6 +38,7 @@ public async Task Serve(string? path = null, Cancel ctx = default)
 	/// <param name="output"> -o, Defaults to `.artifacts/html` </param>
 	/// <param name="pathPrefix"> Specifies the path prefix for urls </param>
 	/// <param name="force"> Force a full rebuild of the destination folder</param>
+	/// <param name="uploadToLinkIndex"> Upload the links.json file to the link index</param>
 	/// <param name="ctx"></param>
 	[Command("generate")]
 	[ConsoleAppFilter<StopwatchFilter>]
@@ -48,6 +48,7 @@ public async Task<int> Generate(
 		string? output = null,
 		string? pathPrefix = null,
 		bool? force = null,
+		bool uploadToLinkIndex = false,
 		Cancel ctx = default
 	)
 	{
@@ -57,11 +58,13 @@ public async Task<int> Generate(
 		{
 			UrlPathPrefix = pathPrefix,
 			Force = force ?? false,
-			Collector = new ConsoleDiagnosticsCollector(logger, githubActionsService)
+			Collector = new ConsoleDiagnosticsCollector(logger, githubActionsService),
 		};
 		var set = new DocumentationSet(context);
 		var generator = new DocumentationGenerator(set, logger);
 		await generator.GenerateAll(ctx);
+		var linksJsonPath = set.LinkReferenceFile.FullName;
+		await linkIndex.UploadFileAsync(linksJsonPath, uploadToLinkIndex);
 		return context.Collector.Errors + context.Collector.Warnings;
 	}
 
@@ -72,6 +75,7 @@ public async Task<int> Generate(
 	/// <param name="output"> -o, Defaults to `.artifacts/html` </param>
 	/// <param name="pathPrefix"> Specifies the path prefix for urls </param>
 	/// <param name="force"> Force a full rebuild of the destination folder</param>
+	/// <param name="uploadToLinkIndex"> Upload the links.json file to the link index</param>
 	/// <param name="ctx"></param>
 	[Command("")]
 	[ConsoleAppFilter<StopwatchFilter>]
@@ -81,7 +85,8 @@ public async Task<int> GenerateDefault(
 		string? output = null,
 		string? pathPrefix = null,
 		bool? force = null,
+		bool uploadToLinkIndex = false,
 		Cancel ctx = default
 	) =>
-		await Generate(path, output, pathPrefix, force, ctx);
+		await Generate(path, output, pathPrefix, force, uploadToLinkIndex, ctx);
 }

src/docs-builder/LinkIndex/ILinkIndex.cs

@@ -0,0 +1,6 @@
+namespace Documentation.Builder.LinkIndex;
+
+public interface ILinkIndex
+{
+    Task UploadFileAsync(string filePath, bool shouldUpload);
+} 

src/docs-builder/LinkIndex/S3LinkIndex.cs

@@ -0,0 +1,64 @@
+using Amazon.S3;
+using Amazon.S3.Transfer;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Configuration;
+
+namespace Documentation.Builder.LinkIndex;
+
+public class S3LinkIndex : ILinkIndex
+{
+    private readonly string _bucketName;
+    private readonly ILogger<S3LinkIndex> _logger;
+
+    public S3LinkIndex(string bucketName, ILoggerFactory loggerFactory)
+    {
+        _bucketName = bucketName;
+        _logger = loggerFactory.CreateLogger<S3LinkIndex>();
+    }
+
+    public async Task UploadFileAsync(string filePath, bool shouldUpload)
+    {
+        if (!shouldUpload)
+        {
+            _logger.LogInformation("Not uploading link index: skip flag is set");
+            return;
+        }
+
+        var githubRef = Environment.GetEnvironmentVariable("GITHUB_REF");
+        if (string.IsNullOrEmpty(githubRef) || githubRef != "refs/heads/main")
+        {
+            _logger.LogWarning("Not uploading link index: GITHUB_REF '{GitHubRef}' is not main branch", githubRef);
+            return;
+        }
+    
+        var s3DestinationPath = DeriveDestinationPath();
+        if (string.IsNullOrEmpty(s3DestinationPath))
+        {
+            _logger.LogWarning("Failed to derive destination path - cannot upload to link index");
+            return;
+        }
+        _logger.LogInformation("Uploading link index {FilePath} to S3://{Bucket}/{DestinationPath}", filePath, _bucketName, s3DestinationPath);
+        using var client = new AmazonS3Client();
+        var fileTransferUtility = new TransferUtility(client);
+        try
+        {
+            await fileTransferUtility.UploadAsync(filePath, _bucketName, s3DestinationPath);
+            _logger.LogInformation("Successfully uploaded link reference {FilePath} to S3://{Bucket}/{DestinationPath}", 
+                filePath, _bucketName, s3DestinationPath);
+        }
+        catch (Exception e)
+        {
+            _logger.LogError(e, "Failed to upload link index {FilePath} to S3://{Bucket}/{DestinationPath}", 
+                filePath, _bucketName, s3DestinationPath);
+            throw;
+        }
+    }
+
+    private static string DeriveDestinationPath()
+    {
+        var repositoryName = Environment.GetEnvironmentVariable("GITHUB_REPOSITORY")?.Split('/').Last();
+        return string.IsNullOrEmpty(repositoryName) 
+            ? string.Empty 
+            : $"{repositoryName}.json";
+    }
+}

src/docs-builder/Program.cs

@@ -9,6 +9,7 @@
 using Elastic.Markdown.Diagnostics;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using Documentation.Builder.LinkIndex;
 
 var services = new ServiceCollection();
 services.AddGitHubActionsCore();
@@ -26,7 +27,8 @@
 });
 services.AddSingleton<DiagnosticsChannel>();
 services.AddSingleton<DiagnosticsCollector>();
-
+services.AddSingleton<S3LinkIndex>(sp => new S3LinkIndex("elastic-docs-link-index", sp.GetRequiredService<ILoggerFactory>()));
+services.AddSingleton<ILinkIndex>(sp => sp.GetRequiredService<S3LinkIndex>());
 
 await using var serviceProvider = services.BuildServiceProvider();
 var logger = serviceProvider.GetRequiredService<ILogger<Program>>();

src/docs-builder/docs-builder.csproj

@@ -19,6 +19,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="AWSSDK.S3" Version="3.7.411.5" />
     <PackageReference Include="ConsoleAppFramework" Version="5.3.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>