Use CrtChecksums explicitly to download S3 objects inside lambda function (#961)

* Use CrtChecksums explicitly to download S3 objects inside lambda function

* Add explicity dep on System.Private.Uri and use default maxkeys for ListObjects call

* use httpclient to download links.json

Author: Mpdreamz

src/infra/docs-lambda-index-publisher/Program.cs

@@ -11,19 +11,40 @@
 using Elastic.Markdown.IO.State;
 using Elastic.Markdown.Links.CrossLinks;
 
-await LambdaBootstrapBuilder.Create(Handler)
-	.Build()
-	.RunAsync();
+const string bucketName = "elastic-docs-link-index";
 
+// await LambdaBootstrapBuilder.Create(Handler)
+// 	.Build()
+// 	.RunAsync();
+
+// Uncomment to test locally without uploading
+await CreateLinkIndex(new AmazonS3Client());
+
+#pragma warning disable CS8321 // Local function is declared but never used
 static async Task<string> Handler(ILambdaContext context)
+#pragma warning restore CS8321 // Local function is declared but never used
 {
 	var sw = Stopwatch.StartNew();
+
 	IAmazonS3 client = new AmazonS3Client();
-	var bucketName = "elastic-docs-link-index";
+	var linkIndex = await CreateLinkIndex(client);
+	if (linkIndex == null)
+		return $"Error encountered on server. getting list of objects.";
+
+	var json = LinkIndex.Serialize(linkIndex);
+
+	using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json));
+	await client.UploadObjectFromStreamAsync(bucketName, "link-index.json", stream, new Dictionary<string, object>(), CancellationToken.None);
+	return $"Finished in {sw}";
+}
+
+
+static async Task<LinkIndex?> CreateLinkIndex(IAmazonS3 client)
+{
 	var request = new ListObjectsV2Request
 	{
 		BucketName = bucketName,
-		MaxKeys = 5
+		MaxKeys = 1000 //default
 	};
 
 	var linkIndex = new LinkIndex
@@ -32,20 +53,23 @@ static async Task<string> Handler(ILambdaContext context)
 	};
 	try
 	{
+		var httpClient = new HttpClient();
 		ListObjectsV2Response response;
 		do
 		{
 			response = await client.ListObjectsV2Async(request, CancellationToken.None);
-			foreach (var obj in response.S3Objects)
+			await Parallel.ForEachAsync(response.S3Objects, async (obj, ctx) =>
 			{
 				if (!obj.Key.StartsWith("elastic/", StringComparison.OrdinalIgnoreCase))
-					continue;
+					return;
 
 				var tokens = obj.Key.Split('/');
 				if (tokens.Length < 3)
-					continue;
+					return;
 
-				var gitReference = await ReadLinkReferenceSha(client, obj);
+				// TODO create a dedicated state file for git configuration
+				// Deserializing all of the links metadata adds significant overhead
+				var gitReference = await ReadLinkReferenceSha(httpClient, obj);
 
 				var repository = tokens[1];
 				var branch = tokens[2];
@@ -67,39 +91,36 @@ static async Task<string> Handler(ILambdaContext context)
 						{ branch, entry }
 					});
 				}
-
-				Console.WriteLine(entry);
-			}
+			});
 
 			// If the response is truncated, set the request ContinuationToken
 			// from the NextContinuationToken property of the response.
 			request.ContinuationToken = response.NextContinuationToken;
 		} while (response.IsTruncated);
 	}
-	catch (AmazonS3Exception ex)
+	catch
 	{
-		return $"Error encountered on server. Message:'{ex.Message}' getting list of objects.";
+		return null;
 	}
 
-	var json = LinkIndex.Serialize(linkIndex);
-
-	using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json));
-	await client.UploadObjectFromStreamAsync(bucketName, "link-index.json", stream, new Dictionary<string, object>(), CancellationToken.None);
-	return $"Finished in {sw}";
+	return linkIndex;
 }
 
-static async Task<string> ReadLinkReferenceSha(IAmazonS3 client, S3Object obj)
+static async Task<string> ReadLinkReferenceSha(HttpClient httpClient, S3Object obj)
 {
 	try
 	{
-		var contents = await client.GetObjectAsync(obj.Key, obj.Key, CancellationToken.None);
-		await using var s = contents.ResponseStream;
-		var linkReference = LinkReference.Deserialize(s);
+		// can not use client getobject since CRT checksum validation requires native code not available in AOT.
+		var tokens = obj.Key.Split('/');
+		var path = Path.Join(tokens[0], tokens[1], "tree", tokens[2], string.Join("/", tokens[3..]));
+		var url = "https://docs-v3-preview.elastic.dev/" + path;
+		var json = await httpClient.GetStringAsync(new Uri(url));
+
+		var linkReference = LinkReference.Deserialize(json);
 		return linkReference.Origin.Ref;
 	}
-	catch (Exception e)
+	catch
 	{
-		Console.WriteLine(e);
 		// it's important we don't fail here we need to fallback gracefully from this so we can fix the root cause
 		// of why a repository is not reporting its git reference properly
 		return "unknown";

src/infra/docs-lambda-index-publisher/aws-lambda-tools-defaults.json

@@ -5,8 +5,8 @@
     "dotnet lambda help",
     "All the command line options for the Lambda command can be specified in this file."
   ],
-  "profile": "",
-  "region": "",
+  "profile": "default",
+  "region": "us-west-2",
   "configuration": "Release",
   "function-runtime": "provided.al2",
   "function-memory-size": 512,