feature/publish links index lamdba

Directory.Build.props

@@ -31,4 +31,4 @@
   <ItemGroup>
     <PackageReference Include="MinVer" Version="6.0.0" PrivateAssets="all" />
   </ItemGroup>
-</Project>
+</Project>

docs-builder.sln

@@ -71,6 +71,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "update-reference-index", "u
 		actions\update-reference-index\action.yml = actions\update-reference-index\action.yml
 	EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs-lambda-index-publisher", "src\docs-lambda-index-publisher\docs-lambda-index-publisher.csproj", "{C559D52D-100B-4B2B-BE87-2344D835761D}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -117,6 +119,10 @@ Global
 		{4CCE599A-B9FE-4DF2-8763-34CF0A99D4AA}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{4CCE599A-B9FE-4DF2-8763-34CF0A99D4AA}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{4CCE599A-B9FE-4DF2-8763-34CF0A99D4AA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C559D52D-100B-4B2B-BE87-2344D835761D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C559D52D-100B-4B2B-BE87-2344D835761D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C559D52D-100B-4B2B-BE87-2344D835761D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C559D52D-100B-4B2B-BE87-2344D835761D}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{4D198E25-C211-41DC-9E84-B15E89BD7048} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
@@ -132,5 +138,6 @@ Global
 		{6E2ED6CC-AFC1-4E58-965D-6AEC500EBB46} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
 		{6554F917-73CE-4B3D-9101-F28EAA762C6B} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
 		{9FEC15F6-13F8-40B1-A66A-EB054E49E680} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
+		{C559D52D-100B-4B2B-BE87-2344D835761D} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
 	EndGlobalSection
 EndGlobal

src/Elastic.Markdown/SourceGenerationContext.cs

@@ -17,4 +17,4 @@ namespace Elastic.Markdown;
 [JsonSerializable(typeof(GitCheckoutInformation))]
 [JsonSerializable(typeof(LinkIndex))]
 [JsonSerializable(typeof(LinkIndexEntry))]
-internal sealed partial class SourceGenerationContext : JsonSerializerContext;
+public sealed partial class SourceGenerationContext : JsonSerializerContext;

src/docs-lambda-index-publisher/Program.cs

@@ -0,0 +1,82 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Diagnostics;
+using System.Text;
+using Amazon.Lambda.Core;
+using Amazon.Lambda.RuntimeSupport;
+using Amazon.S3;
+using Amazon.S3.Model;
+using Elastic.Markdown.CrossLinks;
+
+await LambdaBootstrapBuilder.Create(Handler)
+	.Build()
+	.RunAsync();
+
+static async Task<string> Handler(ILambdaContext context)
+{
+	var sw = Stopwatch.StartNew();
+	IAmazonS3 client = new AmazonS3Client();
+	var bucketName = "elastic-docs-link-index";
+	var request = new ListObjectsV2Request
+	{
+		BucketName = bucketName,
+		MaxKeys = 5
+	};
+
+	var linkIndex = new LinkIndex
+	{
+		Repositories = []
+	};
+	try
+	{
+		ListObjectsV2Response response;
+		do
+		{
+			response = await client.ListObjectsV2Async(request, CancellationToken.None);
+			foreach (var obj in response.S3Objects)
+			{
+				if (!obj.Key.StartsWith("elastic/", StringComparison.OrdinalIgnoreCase))
+					continue;
+
+				var tokens = obj.Key.Split('/');
+				if (tokens.Length < 3)
+					continue;
+
+				var repository = tokens[1];
+				var branch = tokens[2];
+
+				var entry = new LinkIndexEntry
+				{
+					Repository = repository,
+					Branch = branch,
+					ETag = obj.ETag.Trim('"'),
+					Path = obj.Key
+				};
+				if (linkIndex.Repositories.TryGetValue(repository, out var existingEntry))
+					existingEntry[branch] = entry;
+				else
+					linkIndex.Repositories.Add(repository, new Dictionary<string, LinkIndexEntry>
+					{
+						{ branch, entry }
+					});
+				Console.WriteLine(entry);
+			}
+
+			// If the response is truncated, set the request ContinuationToken
+			// from the NextContinuationToken property of the response.
+			request.ContinuationToken = response.NextContinuationToken;
+		} while (response.IsTruncated);
+	}
+	catch (AmazonS3Exception ex)
+	{
+		return $"Error encountered on server. Message:'{ex.Message}' getting list of objects.";
+	}
+
+	var json = LinkIndex.Serialize(linkIndex);
+
+	using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json));
+	await client.UploadObjectFromStreamAsync(bucketName, "link-index.json", stream, new Dictionary<string, object>(), CancellationToken.None);
+	return $"Finished in {sw}";
+}

src/docs-lambda-index-publisher/README.md

@@ -0,0 +1,22 @@
+# Index Registry Update Lambda Function
+
+From a linux `x86_64` machine you can use the followint to build a AOT binary that will run
+
+on a vanilla `Amazon Linux 2023` without any dependencies.
+
+```bash
+docker build . -t publish-links-index:latest -f src/docs-lambda-index-publisher/lambda.DockerFile
+```
+
+Then you can copy the published artifacts from the image using:
+
+```bash
+docker cp (docker create --name tc publish-links-index:latest):/app/.artifacts/publish ./.artifacts && docker rm tc
+```
+
+The `bootstrap` binary should now be available under:
+
+```
+.artifacts/publish/docs-lambda-index-publisher/release_linux-x64/bootstrap
+```
+

src/docs-lambda-index-publisher/aws-lambda-tools-defaults.json

@@ -0,0 +1,15 @@
+{
+  "Information": [
+    "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.",
+    "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.",
+    "dotnet lambda help",
+    "All the command line options for the Lambda command can be specified in this file."
+  ],
+  "profile": "",
+  "region": "",
+  "configuration": "Release",
+  "function-runtime": "provided.al2",
+  "function-memory-size": 512,
+  "function-timeout": 180,
+  "function-handler": "Elastic.Documentation.Lambda.LinkIndexUploader"
+}

src/docs-lambda-index-publisher/docs-lambda-index-publisher.csproj

@@ -0,0 +1,31 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net9.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <InvariantGlobalization>true</InvariantGlobalization>
+
+    <AssemblyName>bootstrap</AssemblyName>
+    <AWSProjectType>Lambda</AWSProjectType>
+
+    <IsPublishable>true</IsPublishable>
+    <PublishAot>true</PublishAot>
+    <PublishTrimmed>true</PublishTrimmed>
+    <EnableSdkContainerSupport>true</EnableSdkContainerSupport>
+    <TrimmerSingleWarn>false</TrimmerSingleWarn>
+    <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
+
+    <RootNamespace>Elastic.Documentation.Lambda.LinkIndexUploader</RootNamespace>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Amazon.Lambda.RuntimeSupport" Version="1.12.3"/>
+    <PackageReference Include="Amazon.Lambda.Core" Version="2.5.0"/>
+    <PackageReference Include="Amazon.Lambda.Serialization.SystemTextJson" Version="2.4.4"/>
+    <PackageReference Include="AWSSDK.S3" Version="3.7.414.5"/>
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\Elastic.Markdown\Elastic.Markdown.csproj"/>
+  </ItemGroup>
+</Project>
+

src/docs-lambda-index-publisher/lambda.DockerFile

@@ -0,0 +1,35 @@
+FROM public.ecr.aws/amazonlinux/amazonlinux:2023 AS base
+
+ARG TARGETARCH
+ARG TARGETOS
+
+WORKDIR /app
+
+RUN rpm --import https://packages.microsoft.com/keys/microsoft.asc
+
+RUN curl -o /etc/yum.repos.d/microsoft-prod.repo https://packages.microsoft.com/config/fedora/39/prod.repo
+
+RUN dnf update -y
+RUN dnf install -y dotnet-sdk-9.0
+RUN dnf install -y npm
+RUN dnf install -y git
+RUN dnf install -y clang
+
+#COPY ./src/ ./src/
+#COPY ./tests/ ./tests/
+#COPY ./docs/*.csproj ./docs/
+#COPY ./.github/*.csproj ./.github/
+#COPY ./build/*.fsproj ./build/
+#COPY ./*.sln ./
+
+COPY . .
+
+ENV DOTNET_NOLOGO=true
+ENV DOTNET_CLI_TELEMETRY_OPTOUT=true
+
+RUN arch=$TARGETARCH \
+    && if [ "$arch" = "amd64" ]; then arch="x64"; fi \
+    && echo $TARGETOS-$arch > /tmp/rid
+
+RUN dotnet publish src/docs-lambda-index-publisher -r linux-x64 -c Release
+