diff --git a/Directory.Build.props b/Directory.Build.props index a49f44bc9..d2138cdc9 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -31,4 +31,4 @@ - \ No newline at end of file + diff --git a/docs-builder.sln b/docs-builder.sln index 92a5e9bda..d6f0c1f8e 100644 --- a/docs-builder.sln +++ b/docs-builder.sln @@ -71,6 +71,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "update-reference-index", "u actions\update-reference-index\action.yml = actions\update-reference-index\action.yml EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs-lambda-index-publisher", "src\docs-lambda-index-publisher\docs-lambda-index-publisher.csproj", "{C559D52D-100B-4B2B-BE87-2344D835761D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -117,6 +119,10 @@ Global {4CCE599A-B9FE-4DF2-8763-34CF0A99D4AA}.Debug|Any CPU.Build.0 = Debug|Any CPU {4CCE599A-B9FE-4DF2-8763-34CF0A99D4AA}.Release|Any CPU.ActiveCfg = Release|Any CPU {4CCE599A-B9FE-4DF2-8763-34CF0A99D4AA}.Release|Any CPU.Build.0 = Release|Any CPU + {C559D52D-100B-4B2B-BE87-2344D835761D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C559D52D-100B-4B2B-BE87-2344D835761D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C559D52D-100B-4B2B-BE87-2344D835761D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C559D52D-100B-4B2B-BE87-2344D835761D}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {4D198E25-C211-41DC-9E84-B15E89BD7048} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A} @@ -132,5 +138,6 @@ Global {6E2ED6CC-AFC1-4E58-965D-6AEC500EBB46} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7} {6554F917-73CE-4B3D-9101-F28EAA762C6B} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7} {9FEC15F6-13F8-40B1-A66A-EB054E49E680} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7} + {C559D52D-100B-4B2B-BE87-2344D835761D} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A} EndGlobalSection EndGlobal diff --git a/src/Elastic.Markdown/SourceGenerationContext.cs b/src/Elastic.Markdown/SourceGenerationContext.cs index 8e1769b96..7b07758b8 100644 --- a/src/Elastic.Markdown/SourceGenerationContext.cs +++ b/src/Elastic.Markdown/SourceGenerationContext.cs @@ -17,4 +17,4 @@ namespace Elastic.Markdown; [JsonSerializable(typeof(GitCheckoutInformation))] [JsonSerializable(typeof(LinkIndex))] [JsonSerializable(typeof(LinkIndexEntry))] -internal sealed partial class SourceGenerationContext : JsonSerializerContext; +public sealed partial class SourceGenerationContext : JsonSerializerContext; diff --git a/src/docs-lambda-index-publisher/Program.cs b/src/docs-lambda-index-publisher/Program.cs new file mode 100644 index 000000000..1507f9dd6 --- /dev/null +++ b/src/docs-lambda-index-publisher/Program.cs @@ -0,0 +1,82 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Diagnostics; +using System.Text; +using Amazon.Lambda.Core; +using Amazon.Lambda.RuntimeSupport; +using Amazon.S3; +using Amazon.S3.Model; +using Elastic.Markdown.CrossLinks; + +await LambdaBootstrapBuilder.Create(Handler) + .Build() + .RunAsync(); + +static async Task Handler(ILambdaContext context) +{ + var sw = Stopwatch.StartNew(); + IAmazonS3 client = new AmazonS3Client(); + var bucketName = "elastic-docs-link-index"; + var request = new ListObjectsV2Request + { + BucketName = bucketName, + MaxKeys = 5 + }; + + var linkIndex = new LinkIndex + { + Repositories = [] + }; + try + { + ListObjectsV2Response response; + do + { + response = await client.ListObjectsV2Async(request, CancellationToken.None); + foreach (var obj in response.S3Objects) + { + if (!obj.Key.StartsWith("elastic/", StringComparison.OrdinalIgnoreCase)) + continue; + + var tokens = obj.Key.Split('/'); + if (tokens.Length < 3) + continue; + + var repository = tokens[1]; + var branch = tokens[2]; + + var entry = new LinkIndexEntry + { + Repository = repository, + Branch = branch, + ETag = obj.ETag.Trim('"'), + Path = obj.Key + }; + if (linkIndex.Repositories.TryGetValue(repository, out var existingEntry)) + existingEntry[branch] = entry; + else + linkIndex.Repositories.Add(repository, new Dictionary + { + { branch, entry } + }); + Console.WriteLine(entry); + } + + // If the response is truncated, set the request ContinuationToken + // from the NextContinuationToken property of the response. + request.ContinuationToken = response.NextContinuationToken; + } while (response.IsTruncated); + } + catch (AmazonS3Exception ex) + { + return $"Error encountered on server. Message:'{ex.Message}' getting list of objects."; + } + + var json = LinkIndex.Serialize(linkIndex); + + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json)); + await client.UploadObjectFromStreamAsync(bucketName, "link-index.json", stream, new Dictionary(), CancellationToken.None); + return $"Finished in {sw}"; +} diff --git a/src/docs-lambda-index-publisher/README.md b/src/docs-lambda-index-publisher/README.md new file mode 100644 index 000000000..1c5abc0a1 --- /dev/null +++ b/src/docs-lambda-index-publisher/README.md @@ -0,0 +1,22 @@ +# Index Registry Update Lambda Function + +From a linux `x86_64` machine you can use the followint to build a AOT binary that will run + +on a vanilla `Amazon Linux 2023` without any dependencies. + +```bash +docker build . -t publish-links-index:latest -f src/docs-lambda-index-publisher/lambda.DockerFile +``` + +Then you can copy the published artifacts from the image using: + +```bash +docker cp (docker create --name tc publish-links-index:latest):/app/.artifacts/publish ./.artifacts && docker rm tc +``` + +The `bootstrap` binary should now be available under: + +``` +.artifacts/publish/docs-lambda-index-publisher/release_linux-x64/bootstrap +``` + diff --git a/src/docs-lambda-index-publisher/aws-lambda-tools-defaults.json b/src/docs-lambda-index-publisher/aws-lambda-tools-defaults.json new file mode 100644 index 000000000..e4e11eed8 --- /dev/null +++ b/src/docs-lambda-index-publisher/aws-lambda-tools-defaults.json @@ -0,0 +1,15 @@ +{ + "Information": [ + "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.", + "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.", + "dotnet lambda help", + "All the command line options for the Lambda command can be specified in this file." + ], + "profile": "", + "region": "", + "configuration": "Release", + "function-runtime": "provided.al2", + "function-memory-size": 512, + "function-timeout": 180, + "function-handler": "Elastic.Documentation.Lambda.LinkIndexUploader" +} \ No newline at end of file diff --git a/src/docs-lambda-index-publisher/docs-lambda-index-publisher.csproj b/src/docs-lambda-index-publisher/docs-lambda-index-publisher.csproj new file mode 100644 index 000000000..949563fc6 --- /dev/null +++ b/src/docs-lambda-index-publisher/docs-lambda-index-publisher.csproj @@ -0,0 +1,31 @@ + + + Exe + net9.0 + enable + enable + true + + bootstrap + Lambda + + true + true + true + true + false + Linux + + Elastic.Documentation.Lambda.LinkIndexUploader + + + + + + + + + + + + diff --git a/src/docs-lambda-index-publisher/lambda.DockerFile b/src/docs-lambda-index-publisher/lambda.DockerFile new file mode 100644 index 000000000..b26032d25 --- /dev/null +++ b/src/docs-lambda-index-publisher/lambda.DockerFile @@ -0,0 +1,35 @@ +FROM public.ecr.aws/amazonlinux/amazonlinux:2023 AS base + +ARG TARGETARCH +ARG TARGETOS + +WORKDIR /app + +RUN rpm --import https://packages.microsoft.com/keys/microsoft.asc + +RUN curl -o /etc/yum.repos.d/microsoft-prod.repo https://packages.microsoft.com/config/fedora/39/prod.repo + +RUN dnf update -y +RUN dnf install -y dotnet-sdk-9.0 +RUN dnf install -y npm +RUN dnf install -y git +RUN dnf install -y clang + +#COPY ./src/ ./src/ +#COPY ./tests/ ./tests/ +#COPY ./docs/*.csproj ./docs/ +#COPY ./.github/*.csproj ./.github/ +#COPY ./build/*.fsproj ./build/ +#COPY ./*.sln ./ + +COPY . . + +ENV DOTNET_NOLOGO=true \ + DOTNET_CLI_TELEMETRY_OPTOUT=true + +RUN arch=$TARGETARCH \ + && if [ "$arch" = "amd64" ]; then arch="x64"; fi \ + && echo $TARGETOS-$arch > /tmp/rid + +RUN dotnet publish src/docs-lambda-index-publisher -r linux-x64 -c Release +