From 53709d13cb729e2a321ce75fcf490d7e356b2727 Mon Sep 17 00:00:00 2001 From: sobrab <170367248+sobrab@users.noreply.github.com> Date: Tue, 7 Jan 2025 16:50:22 +0200 Subject: [PATCH 1/4] added a java example for creating an S3 Object Lambda --- java/s3-object-lambda/.gitignore | 13 + java/s3-object-lambda/README.md | 75 ++++++ java/s3-object-lambda/infra/.gitignore | 13 + java/s3-object-lambda/infra/cdk.json | 76 ++++++ java/s3-object-lambda/infra/pom.xml | 73 ++++++ .../java/com/myorg/S3ObjectLambdaApp.java | 20 ++ .../java/com/myorg/S3ObjectLambdaStack.java | 155 +++++++++++ .../java/com/myorg/S3ObjectLambdaAppTest.java | 36 +++ .../src/test/java/com/myorg/TestUtils.java | 247 ++++++++++++++++++ java/s3-object-lambda/lambda/.gitignore | 13 + java/s3-object-lambda/lambda/pom.xml | 119 +++++++++ .../com/myorg/S3ObjectLambdaTransformer.java | 92 +++++++ .../com/myorg/model/TransformedObject.java | 16 ++ .../myorg/S3ObjectLambdaTransformerTest.java | 65 +++++ 14 files changed, 1013 insertions(+) create mode 100644 java/s3-object-lambda/.gitignore create mode 100644 java/s3-object-lambda/README.md create mode 100644 java/s3-object-lambda/infra/.gitignore create mode 100644 java/s3-object-lambda/infra/cdk.json create mode 100644 java/s3-object-lambda/infra/pom.xml create mode 100644 java/s3-object-lambda/infra/src/main/java/com/myorg/S3ObjectLambdaApp.java create mode 100644 java/s3-object-lambda/infra/src/main/java/com/myorg/S3ObjectLambdaStack.java create mode 100644 java/s3-object-lambda/infra/src/test/java/com/myorg/S3ObjectLambdaAppTest.java create mode 100644 java/s3-object-lambda/infra/src/test/java/com/myorg/TestUtils.java create mode 100644 java/s3-object-lambda/lambda/.gitignore create mode 100644 java/s3-object-lambda/lambda/pom.xml create mode 100644 java/s3-object-lambda/lambda/src/main/java/com/myorg/S3ObjectLambdaTransformer.java create mode 100644 java/s3-object-lambda/lambda/src/main/java/com/myorg/model/TransformedObject.java create mode 100644 java/s3-object-lambda/lambda/src/test/java/com/myorg/S3ObjectLambdaTransformerTest.java diff --git a/java/s3-object-lambda/.gitignore b/java/s3-object-lambda/.gitignore new file mode 100644 index 0000000000..1db21f1629 --- /dev/null +++ b/java/s3-object-lambda/.gitignore @@ -0,0 +1,13 @@ +.classpath.txt +target +.classpath +.project +.idea +.settings +.vscode +*.iml + +# CDK asset staging directory +.cdk.staging +cdk.out + diff --git a/java/s3-object-lambda/README.md b/java/s3-object-lambda/README.md new file mode 100644 index 0000000000..4b8e8dee39 --- /dev/null +++ b/java/s3-object-lambda/README.md @@ -0,0 +1,75 @@ +# S3 Object Lambda + + + +--- +![Stability: Stable](https://img.shields.io/badge/stability-Stable-success.svg?style=for-the-badge) +> **This is a stable example. It should successfully build out of the box** +> +> This example is built on Construct Libraries marked "Stable" and does not have any infrastructure prerequisites to build. +--- + + + +## Overview + +This is a Java CDK example to create a [S3 Object Lambda](https://docs.aws.amazon.com/AmazonS3/latest/userguide/transforming-objects.html). +The main cloud infrastructure resources created with this example are: +- an S3 Bucket; +- a S3 Access Point; +- an S3 Object Lambda Access Point; +- a Lambda Function to process the GET object requests. + +Once this example is deployed, any object uploaded in the bucket created will be available through the S3 Object Lambda Access Point. +When trying to access an object through the S3 Object Lambda Access Point, a metadata output generated by the lambda function will be returned. +The object metadata output generated by the lambda is similar to the following: +```json +{ + "metadata": { + "length": 1048576, + "md5": "b6d81b360a5672d80c27430f39153e2c", + "sha1": "3b71f43ff30f4b15b5cd85dd9e95ebc7e84eb5a3", + "sha256": "30e14955ebf1352266dc2ff8067e68104607e750abb9d3b36582b8af909fcb58" + } +} +``` + +## Build + +**To build this example, you need to be in this example's [`infra`](infra) directory.** Then run the following: +```bash + npm install -g aws-cdk + npm install + cdk synth +``` +> [!NOTE] +> In order to build this project you need [Docker](https://www.docker.com/) (and [Docker Desktop](https://www.docker.com/products/docker-desktop/) if you want to use a GUI) installed. +> Moreover, the Docker daemon must be running while you execute the build or deploy commands. +> Unless the Docker daemon is running the Maven build of the lambda function ([`S3ObjectLambdaTransformer`](lambda/src/main/java/com/myorg/S3ObjectLambdaTransformer.java)) will fail. +> This is required because the CDK will build the code for the lambda function locally in a Docker container. +> You should also create a **virtual file share** for the local `/tmp` directory (see [Synchronized file shares](https://docs.docker.com/desktop/synchronized-file-sharing/)). +> This can be done from Docker Desktop (`Settings>Resources>File sharing`). + +> [!NOTE] +> When you build this project for the first time, the required Docker image ([sam/build-java17](https://gallery.ecr.aws/sam/build-java17)) will be pulled from AWS ECR. +> This will cause the first build to be slower. +> That is unless the required container image is already present on the system. + +## Deploy + +To deploy this example, you need to be in this example's [`infra`](infra) directory and run `cdk deploy`. +This will deploy / redeploy the Stack to AWS. +After the CDK deployment is successful, the URL of the S3 Object Lambda Access Point (the output named `S3ObjectLambdaStack.s3ObjectLambdaAccessPointUrl`) will be available in the terminal console. +At this point, this URL can be used for testing. +Copy the URL and paste it in the address bar of a browser. +This will take you to the access point in the AWS management console (you need to be authenticated for that). +From here you can access an S3 object to get the corresponding metadata (you need to have objects in the bucket for that). + +## Useful commands + + * `mvn package` compile and run tests + * `cdk ls` list all stacks in the app + * `cdk synth` emits the synthesized CloudFormation template + * `cdk deploy` deploy this stack to your default AWS account/region + * `cdk diff` compare deployed stack with current state + * `cdk docs` open CDK documentation diff --git a/java/s3-object-lambda/infra/.gitignore b/java/s3-object-lambda/infra/.gitignore new file mode 100644 index 0000000000..1db21f1629 --- /dev/null +++ b/java/s3-object-lambda/infra/.gitignore @@ -0,0 +1,13 @@ +.classpath.txt +target +.classpath +.project +.idea +.settings +.vscode +*.iml + +# CDK asset staging directory +.cdk.staging +cdk.out + diff --git a/java/s3-object-lambda/infra/cdk.json b/java/s3-object-lambda/infra/cdk.json new file mode 100644 index 0000000000..9f9fb58cd5 --- /dev/null +++ b/java/s3-object-lambda/infra/cdk.json @@ -0,0 +1,76 @@ +{ + "app": "mvn -e -q compile exec:java", + "watch": { + "include": [ + "**" + ], + "exclude": [ + "README.md", + "cdk*.json", + "target", + "pom.xml", + "src/test" + ] + }, + "context": { + "@aws-cdk/aws-lambda:recognizeLayerVersion": true, + "@aws-cdk/core:checkSecretUsage": true, + "@aws-cdk/core:target-partitions": [ + "aws", + "aws-cn" + ], + "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, + "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, + "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, + "@aws-cdk/aws-iam:minimizePolicies": true, + "@aws-cdk/core:validateSnapshotRemovalPolicy": true, + "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, + "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, + "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, + "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, + "@aws-cdk/core:enablePartitionLiterals": true, + "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true, + "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true, + "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true, + "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true, + "@aws-cdk/aws-route53-patters:useCertificate": true, + "@aws-cdk/customresources:installLatestAwsSdkDefault": false, + "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true, + "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true, + "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true, + "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true, + "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true, + "@aws-cdk/aws-redshift:columnId": true, + "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true, + "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true, + "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true, + "@aws-cdk/aws-kms:aliasNameRef": true, + "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true, + "@aws-cdk/core:includePrefixInUniqueNameGeneration": true, + "@aws-cdk/aws-efs:denyAnonymousAccess": true, + "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true, + "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true, + "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true, + "@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true, + "@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true, + "@aws-cdk/aws-rds:preventRenderingDeprecatedCredentials": true, + "@aws-cdk/aws-codepipeline-actions:useNewDefaultBranchForCodeCommitSource": true, + "@aws-cdk/aws-cloudwatch-actions:changeLambdaPermissionLogicalIdForLambdaAction": true, + "@aws-cdk/aws-codepipeline:crossAccountKeysDefaultValueToFalse": true, + "@aws-cdk/aws-codepipeline:defaultPipelineTypeToV2": true, + "@aws-cdk/aws-kms:reduceCrossAccountRegionPolicyScope": true, + "@aws-cdk/aws-eks:nodegroupNameAttribute": true, + "@aws-cdk/aws-ec2:ebsDefaultGp3Volume": true, + "@aws-cdk/aws-ecs:removeDefaultDeploymentAlarm": true, + "@aws-cdk/custom-resources:logApiResponseDataPropertyTrueDefault": false, + "@aws-cdk/aws-s3:keepNotificationInImportedBucket": false, + "@aws-cdk/aws-ecs:reduceEc2FargateCloudWatchPermissions": true, + "@aws-cdk/aws-dynamodb:resourcePolicyPerReplica": true, + "@aws-cdk/aws-ec2:ec2SumTImeoutEnabled": true, + "@aws-cdk/aws-appsync:appSyncGraphQLAPIScopeLambdaPermission": true, + "@aws-cdk/aws-rds:setCorrectValueForDatabaseInstanceReadReplicaInstanceResourceId": true, + "@aws-cdk/core:cfnIncludeRejectComplexResourceUpdateCreatePolicyIntrinsics": true, + "@aws-cdk/aws-lambda-nodejs:sdkV3ExcludeSmithyPackages": true, + "@aws-cdk/aws-stepfunctions-tasks:fixRunEcsTaskPolicy": true + } +} diff --git a/java/s3-object-lambda/infra/pom.xml b/java/s3-object-lambda/infra/pom.xml new file mode 100644 index 0000000000..5efd178559 --- /dev/null +++ b/java/s3-object-lambda/infra/pom.xml @@ -0,0 +1,73 @@ + + + 4.0.0 + + com.myorg.s3-object-lambda + infra + infra + 0.1 + + + UTF-8 + 17 + 17 + 2.171.1 + [10.0.0,11.0.0) + 5.7.1 + 1.18.34 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.11.0 + + ${java.version} + ${java.version} + ${java.version} + + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + com.myorg.S3ObjectLambdaApp + + + + + + + + + software.amazon.awscdk + aws-cdk-lib + ${cdk.version} + + + + software.constructs + constructs + ${constructs.version} + + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + + org.projectlombok + lombok + ${lombok.version} + provided + + + diff --git a/java/s3-object-lambda/infra/src/main/java/com/myorg/S3ObjectLambdaApp.java b/java/s3-object-lambda/infra/src/main/java/com/myorg/S3ObjectLambdaApp.java new file mode 100644 index 0000000000..33cc6613f2 --- /dev/null +++ b/java/s3-object-lambda/infra/src/main/java/com/myorg/S3ObjectLambdaApp.java @@ -0,0 +1,20 @@ +package com.myorg; + +import software.amazon.awscdk.App; +import software.amazon.awscdk.StackProps; + +public class S3ObjectLambdaApp extends App { + + public static void main(final String... args) { + var app = new S3ObjectLambdaApp(); + var stackProps = StackProps.builder().build(); + app.createStack(stackProps); + app.synth(); + } + + public S3ObjectLambdaStack createStack(StackProps stackProps) { + return new S3ObjectLambdaStack(this, "S3ObjectLambdaStack", stackProps); + } + +} + diff --git a/java/s3-object-lambda/infra/src/main/java/com/myorg/S3ObjectLambdaStack.java b/java/s3-object-lambda/infra/src/main/java/com/myorg/S3ObjectLambdaStack.java new file mode 100644 index 0000000000..e50d9af2bf --- /dev/null +++ b/java/s3-object-lambda/infra/src/main/java/com/myorg/S3ObjectLambdaStack.java @@ -0,0 +1,155 @@ +package com.myorg; + +import software.amazon.awscdk.*; +import software.amazon.awscdk.services.iam.*; +import software.amazon.awscdk.services.lambda.Code; +import software.amazon.awscdk.services.lambda.Function; +import software.amazon.awscdk.services.lambda.Permission; +import software.amazon.awscdk.services.lambda.Runtime; +import software.amazon.awscdk.services.s3.BlockPublicAccess; +import software.amazon.awscdk.services.s3.Bucket; +import software.amazon.awscdk.services.s3.BucketAccessControl; +import software.amazon.awscdk.services.s3.BucketEncryption; +import software.amazon.awscdk.services.s3.assets.AssetOptions; +import software.constructs.Construct; +import software.amazon.awscdk.services.s3objectlambda.CfnAccessPoint; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static java.util.Collections.singletonList; +import static software.amazon.awscdk.BundlingOutput.ARCHIVED; +import static software.amazon.awscdk.services.s3objectlambda.CfnAccessPoint.*; + +public class S3ObjectLambdaStack extends Stack { + + private static final String S3_ACCESS_POINT_NAME = "s3-access-point"; + private static final String OBJECT_LAMBDA_ACCESS_POINT_NAME = "object-lambda-access-point"; + + public S3ObjectLambdaStack(final Construct scope, final String id, final StackProps props) { + super(scope, id, props); + var accessPoint = "arn:aws:s3:" + Aws.REGION + ":" + Aws.ACCOUNT_ID + ":accesspoint/" + S3_ACCESS_POINT_NAME; + var s3ObjectLambdaBucket = Bucket.Builder.create(this, "S3ObjectLambdaBucket") + .removalPolicy(RemovalPolicy.RETAIN) + .autoDeleteObjects(false) + .accessControl(BucketAccessControl.BUCKET_OWNER_FULL_CONTROL) + .encryption(BucketEncryption.S3_MANAGED) + .blockPublicAccess(BlockPublicAccess.BLOCK_ALL) + .build(); + var s3ObjectLambdaBucketPolicyStatement = PolicyStatement.Builder.create() + .actions(List.of("*")) + .principals(List.of(new AnyPrincipal())) + .resources(List.of( + s3ObjectLambdaBucket.getBucketArn(), + s3ObjectLambdaBucket.arnForObjects("*") + )) + .conditions( + Map.of( + "StringEquals", Map.of( + "s3:DataAccessPointAccount", Aws.ACCOUNT_ID + ) + ) + ) + .build(); + s3ObjectLambdaBucket.addToResourcePolicy(s3ObjectLambdaBucketPolicyStatement); + var s3ObjectLambdaFunction = createS3ObjectLambdaFunction(); + var s3ObjectLambdaFunctionPolicyStatement = PolicyStatement.Builder.create() + .effect(Effect.ALLOW) + .resources(List.of("*")) + .actions(List.of("s3-object-lambda:WriteGetObjectResponse")) + .build(); + s3ObjectLambdaFunction.addToRolePolicy(s3ObjectLambdaFunctionPolicyStatement); + var s3ObjectLambdaFunctionPermission = Permission.builder() + .action("lambda:InvokeFunction") + .principal(new AccountRootPrincipal()) + .sourceAccount(Aws.ACCOUNT_ID) + .build(); + s3ObjectLambdaFunction.addPermission("S3ObjectLambdaPermission", s3ObjectLambdaFunctionPermission); + var s3ObjectLambdaAccessPointPolicyStatement = PolicyStatement.Builder.create() + .sid("S3ObjectLambdaAccessPointPolicyStatement") + .effect(Effect.ALLOW) + .actions(List.of("s3:GetObject")) + .principals(List.of( + new ArnPrincipal(Objects.requireNonNull(s3ObjectLambdaFunction.getRole()).getRoleArn()) + ) + ) + .resources(List.of(accessPoint + "/object/*")) + .build(); + var s3ObjectLambdaAccessPointPolicyDocument = PolicyDocument.Builder.create() + .statements(List.of( + s3ObjectLambdaAccessPointPolicyStatement + )) + .build(); + software.amazon.awscdk.services.s3.CfnAccessPoint.Builder.create(this, "S3ObjectLambdaS3AccessPoint") + .bucket(s3ObjectLambdaBucket.getBucketName()) + .name(S3_ACCESS_POINT_NAME) + .policy(s3ObjectLambdaAccessPointPolicyDocument) + .build(); + var s3ObjectLambdaAccessPoint = CfnAccessPoint.Builder.create(this, "S3ObjectLambdaAccessPoint") + .name(OBJECT_LAMBDA_ACCESS_POINT_NAME) + .objectLambdaConfiguration(ObjectLambdaConfigurationProperty.builder() + .supportingAccessPoint(accessPoint) + .transformationConfigurations(List.of( + TransformationConfigurationProperty.builder() + .actions(List.of("GetObject")) + .contentTransformation( + Map.of( + "AwsLambda", Map.of( + "FunctionArn", s3ObjectLambdaFunction.getFunctionArn() + ) + ) + ) + .build() + ) + ) + .build() + ) + .build(); + CfnOutput.Builder.create(this, "s3ObjectLambdaBucketArn") + .value(s3ObjectLambdaBucket.getBucketArn()) + .build(); + CfnOutput.Builder.create(this, "s3ObjectLambdaFunctionArn") + .value(s3ObjectLambdaFunction.getFunctionArn()) + .build(); + CfnOutput.Builder.create(this, "s3ObjectLambdaAccessPointArn") + .value(s3ObjectLambdaAccessPoint.getAttrArn()) + .build(); + CfnOutput.Builder.create(this, "s3ObjectLambdaAccessPointUrl") + .value("https://console.aws.amazon.com/s3/olap/" + Aws.ACCOUNT_ID + "/" + OBJECT_LAMBDA_ACCESS_POINT_NAME + "?region=" + Aws.REGION) + .build(); + } + + private Function createS3ObjectLambdaFunction() { + List packagingInstructions = List.of( + "/bin/sh", + "-c", + "mvn -e -q clean package && cp /asset-input/target/lambda-1.0-SNAPSHOT.jar /asset-output/" + ); + var builderOptions = BundlingOptions.builder() + .command(packagingInstructions) + .image(Runtime.JAVA_17.getBundlingImage()) + .volumes( + singletonList( + DockerVolume.builder() + .hostPath(System.getProperty("user.home") + "/.m2/") + .containerPath("/root/.m2/") + .build() + )) + .user("root") + .outputType(ARCHIVED) + .build(); + return Function.Builder.create(this, "S3ObjectLambdaFunction") + .runtime(Runtime.JAVA_17) + .functionName("S3ObjectLambdaFunction") + .memorySize(2048) + .code( + Code.fromAsset( + "../lambda/", + AssetOptions.builder().bundling(builderOptions).build() + ) + ) + .handler("com.myorg.S3ObjectLambdaTransformer::handleRequest") + .build(); + } +} diff --git a/java/s3-object-lambda/infra/src/test/java/com/myorg/S3ObjectLambdaAppTest.java b/java/s3-object-lambda/infra/src/test/java/com/myorg/S3ObjectLambdaAppTest.java new file mode 100644 index 0000000000..a84abe1aa1 --- /dev/null +++ b/java/s3-object-lambda/infra/src/test/java/com/myorg/S3ObjectLambdaAppTest.java @@ -0,0 +1,36 @@ +package com.myorg; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import software.amazon.awscdk.StackProps; +import software.amazon.awscdk.assertions.Template; + +import static com.myorg.TestUtils.TestConstants.*; +import static com.myorg.TestUtils.isResourceInStack; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class S3ObjectLambdaAppTest { + + private static S3ObjectLambdaApp s3ObjectLambdaApp; + + @BeforeAll + public static void setUp() { + s3ObjectLambdaApp = new S3ObjectLambdaApp(); + } + + @Test + public void testCreateStack() { + var stackProps = StackProps.builder() + .build(); + var stack = s3ObjectLambdaApp.createStack(stackProps); + var template = Template.fromStack(stack); + assertTrue(isResourceInStack(BUCKET_MATCH_PAIR, template), "The expected S3 bucket is not present in the resources of the stack."); + assertTrue(isResourceInStack(BUCKET_POLICY_MATCH_PAIR, template), "The expected S3 bucket policy is not present in the resources of the stack."); + assertTrue(isResourceInStack(IAM_ROLE_MATCH_PAIR, template), "The expected IAM role for the lambda function is not present in the resources of the stack."); + assertTrue(isResourceInStack(IAM_POLICY_MATCH_PAIR, template), "The expected IAM policy for the lambda function is not present in the resources of the stack."); + assertTrue(isResourceInStack(LAMBDA_FUNCTION_MATCH_PAIR, template), "The expected lambda function is not present in the resources of the stack."); + assertTrue(isResourceInStack(LAMBDA_PERMISSION_MATCH_PAIR, template), "The expected lambda permission is not present in the resources of the stack."); + assertTrue(isResourceInStack(S3_ACCESS_POINT_MATCH_PAIR, template), "The expected S3 access point is not present in the resources of the stack."); + assertTrue(isResourceInStack(S3_OBJECT_LAMBDA_ACCESS_POINT, template), "The expected S3 object lambda access point is not present in the resources of the stack."); + } +} diff --git a/java/s3-object-lambda/infra/src/test/java/com/myorg/TestUtils.java b/java/s3-object-lambda/infra/src/test/java/com/myorg/TestUtils.java new file mode 100644 index 0000000000..bec15a7c00 --- /dev/null +++ b/java/s3-object-lambda/infra/src/test/java/com/myorg/TestUtils.java @@ -0,0 +1,247 @@ +package com.myorg; + +import lombok.Builder; +import org.junit.platform.commons.util.StringUtils; +import software.amazon.awscdk.assertions.Template; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Stream; + +public class TestUtils { + + @Builder(builderClassName = "MatchPairBuilder", setterPrefix = "with") + public record MatchPair(String type, Object props) { + + } + + public static class TestConstants { + public static final MatchPair BUCKET_MATCH_PAIR = MatchPair.builder() + .withType("AWS::S3::Bucket") + .withProps( + Map.of( + "UpdateReplacePolicy", "Retain", + "DeletionPolicy", "Retain", + "Properties", Map.of( + "AccessControl", "BucketOwnerFullControl", + "BucketEncryption", Map.of( + "ServerSideEncryptionConfiguration", List.of( + Map.of( + "ServerSideEncryptionByDefault", Map.of( + "SSEAlgorithm", "AES256" + ) + ) + ) + ), + "PublicAccessBlockConfiguration", Map.of( + "BlockPublicAcls", true, + "BlockPublicPolicy", true, + "IgnorePublicAcls", true, + "RestrictPublicBuckets", true + ) + ) + ) + ) + .build(); + public static final MatchPair BUCKET_POLICY_MATCH_PAIR = MatchPair.builder() + .withType("AWS::S3::BucketPolicy") + .withProps( + Map.of( + "Properties", Map.of( + "PolicyDocument", Map.of( + "Statement", List.of( + Map.of( + "Action", "*", + "Condition", Map.of( + "StringEquals", Map.of( + "s3:DataAccessPointAccount", Map.of( + "Ref", "AWS::AccountId" + ) + ) + ), + "Effect", "Allow", + "Principal", Map.of( + "AWS", "*" + ) + ) + ), + "Version", "2012-10-17" + ) + ) + ) + ) + .build(); + public static final MatchPair IAM_ROLE_MATCH_PAIR = MatchPair.builder() + .withType("AWS::IAM::Role") + .withProps( + Map.of( + "Properties", Map.of( + "AssumeRolePolicyDocument", Map.of( + "Statement", List.of( + Map.of( + "Action", "sts:AssumeRole", + "Effect", "Allow", + "Principal", Map.of( + "Service", "lambda.amazonaws.com" + ) + ) + ), + "Version", "2012-10-17" + ), + "ManagedPolicyArns", List.of( + Map.of( + "Fn::Join", List.of( + "", List.of( + "arn:", + Map.of( + "Ref", "AWS::Partition" + ), + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ) + ) + ) + ) + ) + ) + ) + .build(); + public static final MatchPair IAM_POLICY_MATCH_PAIR = MatchPair.builder() + .withType("AWS::IAM::Policy") + .withProps( + Map.of( + "Properties", Map.of( + "PolicyDocument", Map.of( + "Statement", List.of( + Map.of( + "Action", "s3-object-lambda:WriteGetObjectResponse", + "Effect", "Allow", + "Resource", "*" + ) + ), + "Version", "2012-10-17" + ) + ) + ) + ) + .build(); + public static final MatchPair LAMBDA_FUNCTION_MATCH_PAIR = MatchPair.builder() + .withType("AWS::Lambda::Function") + .withProps( + Map.of( + "Properties", Map.of( + "FunctionName", "S3ObjectLambdaFunction", + "Handler", "com.myorg.S3ObjectLambdaTransformer::handleRequest", + "MemorySize", 2048, + "Runtime", "java17" + ) + ) + ) + .build(); + public static final MatchPair LAMBDA_PERMISSION_MATCH_PAIR = MatchPair.builder() + .withType("AWS::Lambda::Permission") + .withProps( + Map.of( + "Properties", Map.of( + "Action", "lambda:InvokeFunction", + "Principal", Map.of( + "Ref", "AWS::AccountId" + ), + "SourceAccount", Map.of( + "Ref", "AWS::AccountId" + ) + ) + ) + ) + .build(); + public static final MatchPair S3_ACCESS_POINT_MATCH_PAIR = MatchPair.builder() + .withType("AWS::S3::AccessPoint") + .withProps( + Map.of( + "Properties", Map.of( + "Name", "s3-access-point", + "Policy", Map.of( + "Statement", List.of( + Map.of( + "Action", "s3:GetObject", + "Effect", "Allow", + "Resource", Map.of( + "Fn::Join", List.of( + "", + List.of( + "arn:aws:s3:", + Map.of( + "Ref", "AWS::Region" + ), + ":", + Map.of( + "Ref", "AWS::AccountId" + ), + ":accesspoint/s3-access-point/object/*" + ) + ) + ), + "Sid", "S3ObjectLambdaAccessPointPolicyStatement" + ) + ), + "Version", "2012-10-17" + ) + ) + ) + ) + .build(); + public static final MatchPair S3_OBJECT_LAMBDA_ACCESS_POINT = MatchPair.builder() + .withType("AWS::S3ObjectLambda::AccessPoint") + .withProps( + Map.of( + "Properties", Map.of( + "Name", "object-lambda-access-point", + "ObjectLambdaConfiguration", Map.of( + "SupportingAccessPoint", Map.of( + "Fn::Join", List.of( + "", + List.of( + "arn:aws:s3:", + Map.of( + "Ref", "AWS::Region" + ), + ":", + Map.of( + "Ref", "AWS::AccountId" + ), + ":accesspoint/s3-access-point" + ) + ) + ), + "TransformationConfigurations", List.of( + Map.of( + "Actions", List.of( + "GetObject" + ), + "ContentTransformation", Map.of( + "AwsLambda", Map.of() + ) + ) + ) + ) + ) + ) + ) + .build(); + } + + public static boolean isResourceInStack(MatchPair matchPair, Template template) { + return Optional.ofNullable(matchPair) + .filter(pair -> StringUtils.isNotBlank(pair.type)) + .filter(pair -> pair.props != null) + .map(pair -> template.findResources(pair.type, pair.props)) + .map(Map::entrySet) + .map(Set::stream) + .map(Stream::count) + .filter(count -> count == 1) + .isPresent(); + } + + +} diff --git a/java/s3-object-lambda/lambda/.gitignore b/java/s3-object-lambda/lambda/.gitignore new file mode 100644 index 0000000000..1db21f1629 --- /dev/null +++ b/java/s3-object-lambda/lambda/.gitignore @@ -0,0 +1,13 @@ +.classpath.txt +target +.classpath +.project +.idea +.settings +.vscode +*.iml + +# CDK asset staging directory +.cdk.staging +cdk.out + diff --git a/java/s3-object-lambda/lambda/pom.xml b/java/s3-object-lambda/lambda/pom.xml new file mode 100644 index 0000000000..0028d4a954 --- /dev/null +++ b/java/s3-object-lambda/lambda/pom.xml @@ -0,0 +1,119 @@ + + + 4.0.0 + + com.myorg.s3-object-lambda + lambda + lambda + 1.0-SNAPSHOT + + + UTF-8 + 17 + 17 + 1.2.3 + 3.11.0 + 2.29.43 + 1.17.1 + 2.18.0 + 1.18.34 + 5.7.1 + 5.14.2 + + + + + com.amazonaws + aws-lambda-java-core + ${aws-lambda-java-core.version} + + + com.amazonaws + aws-lambda-java-events + ${aws-lambda-java-events.version} + + + software.amazon.awssdk + s3 + ${awssdk-s3} + + + commons-codec + commons-codec + ${commons-codec} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson-databind.version} + + + org.projectlombok + lombok + ${lombok.version} + provided + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + org.mockito + mockito-core + ${mockito.version} + test + + + org.mockito + mockito-junit-jupiter + ${mockito.version} + test + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.11.0 + + ${java.version} + ${java.version} + ${java.version} + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.1 + + + -javaagent:${settings.localRepository}/org/mockito/mockito-core/${mockito.version}/mockito-core-${mockito.version}.jar + -Xshare:off + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.0 + + false + + + + package + + shade + + + + + + + + diff --git a/java/s3-object-lambda/lambda/src/main/java/com/myorg/S3ObjectLambdaTransformer.java b/java/s3-object-lambda/lambda/src/main/java/com/myorg/S3ObjectLambdaTransformer.java new file mode 100644 index 0000000000..d22f2af507 --- /dev/null +++ b/java/s3-object-lambda/lambda/src/main/java/com/myorg/S3ObjectLambdaTransformer.java @@ -0,0 +1,92 @@ +package com.myorg; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.events.S3ObjectLambdaEvent; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.myorg.model.TransformedObject; +import org.apache.commons.codec.digest.DigestUtils; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.WriteGetObjectResponseRequest; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.Optional; + +import static java.net.http.HttpResponse.BodyHandlers.ofInputStream; + +public class S3ObjectLambdaTransformer { + + public void handleRequest(S3ObjectLambdaEvent event, Context context) { + try (var s3Client = S3Client.create()) { + var objectMapper = createObjectMapper(); + log(context, "event: " + writeValue(objectMapper, event)); + var objectContext = event.getGetObjectContext(); + var s3Url = objectContext.getInputS3Url(); + var uri = URI.create(s3Url); + var httpClient = HttpClient.newBuilder().build(); + var httpRequest = HttpRequest.newBuilder(uri).GET().build(); + var response = httpClient.send(httpRequest, ofInputStream()); + var requestBody = RequestBody.empty(); + var responseBodyBytes = readBytes(response); + var writeGetObjectResponseRequestBuilder = WriteGetObjectResponseRequest.builder() + .requestRoute(event.outputRoute()) + .requestToken(event.outputToken()) + .statusCode(response.statusCode()); + if (response.statusCode() == 200) { + var metadata = TransformedObject.Metadata.builder() + .withLength((long) responseBodyBytes.length) + .withMD5(DigestUtils.md5Hex(responseBodyBytes)) + .withSHA1(DigestUtils.sha1Hex(responseBodyBytes)) + .withSHA256(DigestUtils.sha256Hex(responseBodyBytes)) + .build(); + var transformedObject = TransformedObject.builder() + .withMetadata(metadata) + .build(); + log(context, "transformedObject: " + writeValue(objectMapper, transformedObject)); + requestBody = RequestBody.fromString(writeValue(objectMapper, transformedObject)); + } else { + writeGetObjectResponseRequestBuilder + .errorMessage(new String(responseBodyBytes)); + } + s3Client.writeGetObjectResponse(writeGetObjectResponseRequestBuilder.build(), requestBody); + } catch (IOException | InterruptedException e) { + throw new RuntimeException("Error while handling request: " + e.getMessage(), e); + } + } + + private static byte[] readBytes(HttpResponse response) throws IOException { + try (var inputStream = response.body()) { + return inputStream.readAllBytes(); + } + } + + private static void log(Context context, String message) { + if (message != null) { + Optional.ofNullable(context) + .map(Context::getLogger) + .ifPresent(lambdaLogger -> lambdaLogger.log(message)); + } + } + + private static ObjectMapper createObjectMapper() { + var objectMapper = new ObjectMapper(); + objectMapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + return objectMapper; + } + + private static String writeValue(ObjectMapper objectMapper, Object object) { + try { + return objectMapper.writeValueAsString(object); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/java/s3-object-lambda/lambda/src/main/java/com/myorg/model/TransformedObject.java b/java/s3-object-lambda/lambda/src/main/java/com/myorg/model/TransformedObject.java new file mode 100644 index 0000000000..a62bb814d2 --- /dev/null +++ b/java/s3-object-lambda/lambda/src/main/java/com/myorg/model/TransformedObject.java @@ -0,0 +1,16 @@ +package com.myorg.model; + +import lombok.*; + +@Builder(setterPrefix = "with", builderClassName = "TransformedObjectBuilder") +@ToString +@Data +public class TransformedObject { + + @Builder(setterPrefix = "with", builderClassName = "MetadataBuilder") + public record Metadata(Long length, String MD5, String SHA1, String SHA256) { + + } + + private Metadata metadata; +} diff --git a/java/s3-object-lambda/lambda/src/test/java/com/myorg/S3ObjectLambdaTransformerTest.java b/java/s3-object-lambda/lambda/src/test/java/com/myorg/S3ObjectLambdaTransformerTest.java new file mode 100644 index 0000000000..2053bc3e5a --- /dev/null +++ b/java/s3-object-lambda/lambda/src/test/java/com/myorg/S3ObjectLambdaTransformerTest.java @@ -0,0 +1,65 @@ +package com.myorg; + +import com.amazonaws.services.lambda.runtime.events.S3ObjectLambdaEvent; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.junit.jupiter.MockitoExtension; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.WriteGetObjectResponseRequest; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; + +import static com.amazonaws.services.lambda.runtime.events.S3ObjectLambdaEvent.GetObjectContext; +import static java.net.http.HttpResponse.BodyHandlers.ofInputStream; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class S3ObjectLambdaTransformerTest { + + @Mock + private S3Client s3Client; + @Mock + private HttpClient.Builder httpClientBuilder; + @Mock + private HttpClient httpClient; + @Mock + private HttpResponse response; + + private final S3ObjectLambdaTransformer s3ObjectLambdaTransformer = new S3ObjectLambdaTransformer(); + + @Test + void testHandleRequest() throws IOException, InterruptedException { + var s3ObjectContent = "S3 test data."; + var event = S3ObjectLambdaEvent.builder() + .withGetObjectContext( + GetObjectContext.builder() + .withInputS3Url("https://s3-access-point-111111111111.s3-accesspoint.eu-north-1.amazonaws.com/test.txt") + .build() + ) + .build(); + try ( + MockedStatic mockedStaticS3Client = mockStatic(S3Client.class); + MockedStatic mockedStaticHttpClient = mockStatic(HttpClient.class) + ) { + mockedStaticS3Client.when(S3Client::create).thenReturn(s3Client); + mockedStaticHttpClient.when(HttpClient::newBuilder).thenReturn(httpClientBuilder); + when(httpClientBuilder.build()).thenReturn(httpClient); + when(httpClient.send(any(HttpRequest.class), eq(ofInputStream()))).thenReturn(response); + when(response.statusCode()).thenReturn(200); + when(response.body()).thenReturn(new ByteArrayInputStream(s3ObjectContent.getBytes(StandardCharsets.UTF_8))); + s3ObjectLambdaTransformer.handleRequest(event, null); + // test if the writeGetObjectResponse method of the s3Client instance is called to send the transformed object + verify(s3Client).writeGetObjectResponse(any(WriteGetObjectResponseRequest.class), any(RequestBody.class)); + } + } + +} From c08a29c0c9dfa6cfbf4ce7e3eddb47ce082b55d5 Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Thu, 9 Jan 2025 10:29:17 -0600 Subject: [PATCH 2/4] Add comments explaining the code --- .../java/com/myorg/S3ObjectLambdaApp.java | 6 ++ .../java/com/myorg/S3ObjectLambdaStack.java | 59 ++++++++++++--- .../com/myorg/S3ObjectLambdaTransformer.java | 74 ++++++++++++++++--- .../com/myorg/model/TransformedObject.java | 8 ++ 4 files changed, 126 insertions(+), 21 deletions(-) diff --git a/java/s3-object-lambda/infra/src/main/java/com/myorg/S3ObjectLambdaApp.java b/java/s3-object-lambda/infra/src/main/java/com/myorg/S3ObjectLambdaApp.java index 33cc6613f2..9dbe76a49c 100644 --- a/java/s3-object-lambda/infra/src/main/java/com/myorg/S3ObjectLambdaApp.java +++ b/java/s3-object-lambda/infra/src/main/java/com/myorg/S3ObjectLambdaApp.java @@ -3,6 +3,9 @@ import software.amazon.awscdk.App; import software.amazon.awscdk.StackProps; +/** + * Main CDK application class that serves as the entry point for deploying the S3 Object Lambda infrastructure. + */ public class S3ObjectLambdaApp extends App { public static void main(final String... args) { @@ -12,6 +15,9 @@ public static void main(final String... args) { app.synth(); } + /** + * Creates a new instance of the S3ObjectLambdaStack with the specified properties. + */ public S3ObjectLambdaStack createStack(StackProps stackProps) { return new S3ObjectLambdaStack(this, "S3ObjectLambdaStack", stackProps); } diff --git a/java/s3-object-lambda/infra/src/main/java/com/myorg/S3ObjectLambdaStack.java b/java/s3-object-lambda/infra/src/main/java/com/myorg/S3ObjectLambdaStack.java index e50d9af2bf..eb2a9c1a8d 100644 --- a/java/s3-object-lambda/infra/src/main/java/com/myorg/S3ObjectLambdaStack.java +++ b/java/s3-object-lambda/infra/src/main/java/com/myorg/S3ObjectLambdaStack.java @@ -23,13 +23,19 @@ import static software.amazon.awscdk.services.s3objectlambda.CfnAccessPoint.*; public class S3ObjectLambdaStack extends Stack { - private static final String S3_ACCESS_POINT_NAME = "s3-access-point"; private static final String OBJECT_LAMBDA_ACCESS_POINT_NAME = "object-lambda-access-point"; + /** + * Constructs a new S3ObjectLambdaStack. + */ public S3ObjectLambdaStack(final Construct scope, final String id, final StackProps props) { - super(scope, id, props); + super(scope, id, props); + + // Construct the access point ARN using the region, account ID and access point name var accessPoint = "arn:aws:s3:" + Aws.REGION + ":" + Aws.ACCOUNT_ID + ":accesspoint/" + S3_ACCESS_POINT_NAME; + + // Create a new S3 bucket with secure configuration including: var s3ObjectLambdaBucket = Bucket.Builder.create(this, "S3ObjectLambdaBucket") .removalPolicy(RemovalPolicy.RETAIN) .autoDeleteObjects(false) @@ -37,6 +43,8 @@ public S3ObjectLambdaStack(final Construct scope, final String id, final StackPr .encryption(BucketEncryption.S3_MANAGED) .blockPublicAccess(BlockPublicAccess.BLOCK_ALL) .build(); + + // Create bucket policy statement allowing access through access points var s3ObjectLambdaBucketPolicyStatement = PolicyStatement.Builder.create() .actions(List.of("*")) .principals(List.of(new AnyPrincipal())) @@ -52,20 +60,30 @@ public S3ObjectLambdaStack(final Construct scope, final String id, final StackPr ) ) .build(); + + // Attach the policy to the bucket s3ObjectLambdaBucket.addToResourcePolicy(s3ObjectLambdaBucketPolicyStatement); + + // Create the Lambda function that will transform objects var s3ObjectLambdaFunction = createS3ObjectLambdaFunction(); + + // Add permission for Lambda to write GetObject responses for s3 object var s3ObjectLambdaFunctionPolicyStatement = PolicyStatement.Builder.create() .effect(Effect.ALLOW) .resources(List.of("*")) .actions(List.of("s3-object-lambda:WriteGetObjectResponse")) .build(); s3ObjectLambdaFunction.addToRolePolicy(s3ObjectLambdaFunctionPolicyStatement); + + // Add permission for the account root to invoke the Lambda function var s3ObjectLambdaFunctionPermission = Permission.builder() .action("lambda:InvokeFunction") .principal(new AccountRootPrincipal()) .sourceAccount(Aws.ACCOUNT_ID) .build(); s3ObjectLambdaFunction.addPermission("S3ObjectLambdaPermission", s3ObjectLambdaFunctionPermission); + + // Create policy allowing Lambda function to get objects through the access point var s3ObjectLambdaAccessPointPolicyStatement = PolicyStatement.Builder.create() .sid("S3ObjectLambdaAccessPointPolicyStatement") .effect(Effect.ALLOW) @@ -76,16 +94,22 @@ public S3ObjectLambdaStack(final Construct scope, final String id, final StackPr ) .resources(List.of(accessPoint + "/object/*")) .build(); + + // Create policy document containing the access point policy var s3ObjectLambdaAccessPointPolicyDocument = PolicyDocument.Builder.create() .statements(List.of( s3ObjectLambdaAccessPointPolicyStatement )) .build(); + + // Create the S3 access point for direct bucket access software.amazon.awscdk.services.s3.CfnAccessPoint.Builder.create(this, "S3ObjectLambdaS3AccessPoint") .bucket(s3ObjectLambdaBucket.getBucketName()) .name(S3_ACCESS_POINT_NAME) .policy(s3ObjectLambdaAccessPointPolicyDocument) .build(); + + // Create the Object Lambda access point that will transform objects var s3ObjectLambdaAccessPoint = CfnAccessPoint.Builder.create(this, "S3ObjectLambdaAccessPoint") .name(OBJECT_LAMBDA_ACCESS_POINT_NAME) .objectLambdaConfiguration(ObjectLambdaConfigurationProperty.builder() @@ -107,28 +131,39 @@ public S3ObjectLambdaStack(final Construct scope, final String id, final StackPr ) .build(); CfnOutput.Builder.create(this, "s3ObjectLambdaBucketArn") - .value(s3ObjectLambdaBucket.getBucketArn()) + .value(s3ObjectLambdaBucket.getBucketArn()) // Export bucket ARN .build(); CfnOutput.Builder.create(this, "s3ObjectLambdaFunctionArn") - .value(s3ObjectLambdaFunction.getFunctionArn()) + .value(s3ObjectLambdaFunction.getFunctionArn()) // Export Lambda function ARN .build(); CfnOutput.Builder.create(this, "s3ObjectLambdaAccessPointArn") - .value(s3ObjectLambdaAccessPoint.getAttrArn()) + .value(s3ObjectLambdaAccessPoint.getAttrArn()) // Export access point ARN .build(); + + // Create output with Console URL for easy access to the Lambda access point CfnOutput.Builder.create(this, "s3ObjectLambdaAccessPointUrl") .value("https://console.aws.amazon.com/s3/olap/" + Aws.ACCOUNT_ID + "/" + OBJECT_LAMBDA_ACCESS_POINT_NAME + "?region=" + Aws.REGION) .build(); } + /** + * Creates the Lambda function that will process S3 Object Lambda requests. + * This method configures the function's runtime, code, and build process. + * + * @return A Lambda Function construct configured for S3 Object Lambda processing + */ private Function createS3ObjectLambdaFunction() { + // Define Maven packaging commands to build the Lambda function List packagingInstructions = List.of( "/bin/sh", "-c", + // Build the project and copy the JAR to the asset output directory "mvn -e -q clean package && cp /asset-input/target/lambda-1.0-SNAPSHOT.jar /asset-output/" ); + // Configure the bundling options for packaging the Lambda function var builderOptions = BundlingOptions.builder() - .command(packagingInstructions) - .image(Runtime.JAVA_17.getBundlingImage()) + .command(packagingInstructions) // Set the Maven build commands + .image(Runtime.JAVA_17.getBundlingImage()) // Use Java 17 runtime image .volumes( singletonList( DockerVolume.builder() @@ -139,10 +174,12 @@ private Function createS3ObjectLambdaFunction() { .user("root") .outputType(ARCHIVED) .build(); + + // Create the Lambda function with specified configuration return Function.Builder.create(this, "S3ObjectLambdaFunction") - .runtime(Runtime.JAVA_17) - .functionName("S3ObjectLambdaFunction") - .memorySize(2048) + .runtime(Runtime.JAVA_17) // Set Java 17 runtime + .functionName("S3ObjectLambdaFunction") // Set function name + .memorySize(2048) // Allocate 2GB memory .code( Code.fromAsset( "../lambda/", @@ -152,4 +189,4 @@ private Function createS3ObjectLambdaFunction() { .handler("com.myorg.S3ObjectLambdaTransformer::handleRequest") .build(); } -} +} \ No newline at end of file diff --git a/java/s3-object-lambda/lambda/src/main/java/com/myorg/S3ObjectLambdaTransformer.java b/java/s3-object-lambda/lambda/src/main/java/com/myorg/S3ObjectLambdaTransformer.java index d22f2af507..40109e9cb4 100644 --- a/java/s3-object-lambda/lambda/src/main/java/com/myorg/S3ObjectLambdaTransformer.java +++ b/java/s3-object-lambda/lambda/src/main/java/com/myorg/S3ObjectLambdaTransformer.java @@ -21,14 +21,33 @@ import static java.net.http.HttpResponse.BodyHandlers.ofInputStream; +/** + * AWS Lambda function handler that processes S3 Object Lambda requests. + * This class implements the transformation logic for S3 objects accessed + * through + * the Object Lambda Access Point. + */ public class S3ObjectLambdaTransformer { + /** + * Main handler method that processes S3 Object Lambda events. + * This method retrieves the original object from S3, applies transformations, + * and returns the modified object data. + * + * @param event The S3 Object Lambda event containing request details + * @param context The Lambda execution context + */ public void handleRequest(S3ObjectLambdaEvent event, Context context) { try (var s3Client = S3Client.create()) { + // Create JSON mapper and log the incoming event var objectMapper = createObjectMapper(); log(context, "event: " + writeValue(objectMapper, event)); + + // Extract the pre-signed URL from the event context var objectContext = event.getGetObjectContext(); var s3Url = objectContext.getInputS3Url(); + + // Create HTTP client and fetch the original object var uri = URI.create(s3Url); var httpClient = HttpClient.newBuilder().build(); var httpRequest = HttpRequest.newBuilder(uri).GET().build(); @@ -39,48 +58,83 @@ public void handleRequest(S3ObjectLambdaEvent event, Context context) { .requestRoute(event.outputRoute()) .requestToken(event.outputToken()) .statusCode(response.statusCode()); + // Process successful responses (HTTP 200) if (response.statusCode() == 200) { + // Build metadata object with content length and hash values var metadata = TransformedObject.Metadata.builder() - .withLength((long) responseBodyBytes.length) - .withMD5(DigestUtils.md5Hex(responseBodyBytes)) - .withSHA1(DigestUtils.sha1Hex(responseBodyBytes)) - .withSHA256(DigestUtils.sha256Hex(responseBodyBytes)) + .withLength((long) responseBodyBytes.length) // Set content length + .withMD5(DigestUtils.md5Hex(responseBodyBytes)) // Calculate MD5 hash + .withSHA1(DigestUtils.sha1Hex(responseBodyBytes)) // Calculate SHA1 hash + .withSHA256(DigestUtils.sha256Hex(responseBodyBytes)) // Calculate SHA256 hash .build(); + // Create transformed object containing the metadata var transformedObject = TransformedObject.builder() .withMetadata(metadata) .build(); + // Log the transformed object for debugging log(context, "transformedObject: " + writeValue(objectMapper, transformedObject)); requestBody = RequestBody.fromString(writeValue(objectMapper, transformedObject)); - } else { + } else { + // Handle non-200 HTTP responses by setting the error message writeGetObjectResponseRequestBuilder - .errorMessage(new String(responseBodyBytes)); + .errorMessage(new String(responseBodyBytes)); // Convert error response body to string and set as error message } + // Write the final response back to S3 Object Lambda with either transformed object or error details s3Client.writeGetObjectResponse(writeGetObjectResponseRequestBuilder.build(), requestBody); } catch (IOException | InterruptedException e) { - throw new RuntimeException("Error while handling request: " + e.getMessage(), e); - } + // Wrap and rethrow any IO or threading exceptions that occur during processing + throw new RuntimeException("Error while handling request: " + e.getMessage(), e) } } + /** + * Reads all bytes from the input stream of an HTTP response. + * Converts the response stream into a byte array for processing. + * + * @param response HTTP response containing the input stream to read + * @return byte array containing all the data from the input stream + * @throws IOException if an I/O error occurs while reading the stream + */ private static byte[] readBytes(HttpResponse response) throws IOException { try (var inputStream = response.body()) { return inputStream.readAllBytes(); } } + /** + * Logs a message to CloudWatch using the Lambda context logger. + * Prefixes the message with the request ID for tracing purposes. + * + * @param context Lambda execution context containing the logger + * @param message Message to be logged + */ private static void log(Context context, String message) { if (message != null) { Optional.ofNullable(context) - .map(Context::getLogger) - .ifPresent(lambdaLogger -> lambdaLogger.log(message)); + .map(Context::getLogger) + .ifPresent(lambdaLogger -> lambdaLogger.log(message)); } } + /** + * Creates and configures a Jackson ObjectMapper for JSON serialization. + * Enables pretty printing and disables failing on empty beans. + * + * @return Configured ObjectMapper instance + */ private static ObjectMapper createObjectMapper() { var objectMapper = new ObjectMapper(); objectMapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); return objectMapper; } + /** + * Serializes an object to a JSON string using the provided ObjectMapper. + * Handles JsonProcessingException by returning an error message. + * + * @param objectMapper The ObjectMapper to use for serialization + * @param object The object to serialize + * @return JSON string representation of the object or error message + */ private static String writeValue(ObjectMapper objectMapper, Object object) { try { return objectMapper.writeValueAsString(object); diff --git a/java/s3-object-lambda/lambda/src/main/java/com/myorg/model/TransformedObject.java b/java/s3-object-lambda/lambda/src/main/java/com/myorg/model/TransformedObject.java index a62bb814d2..fc6019cdab 100644 --- a/java/s3-object-lambda/lambda/src/main/java/com/myorg/model/TransformedObject.java +++ b/java/s3-object-lambda/lambda/src/main/java/com/myorg/model/TransformedObject.java @@ -2,11 +2,19 @@ import lombok.*; +/** + * Represents a transformed S3 object with its metadata. + * This class uses Lombok annotations for builder pattern, toString, and data methods generation. + */ @Builder(setterPrefix = "with", builderClassName = "TransformedObjectBuilder") @ToString @Data public class TransformedObject { + /** + * Record class representing the metadata of a transformed object. + * Contains various hash values and the object length for verification purposes. + */ @Builder(setterPrefix = "with", builderClassName = "MetadataBuilder") public record Metadata(Long length, String MD5, String SHA1, String SHA256) { From b14147bed3e8564ad6407a37f0a83fdcab6323dc Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Thu, 9 Jan 2025 11:15:55 -0600 Subject: [PATCH 3/4] Fix line ending --- .../com/myorg/S3ObjectLambdaTransformer.java | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/java/s3-object-lambda/lambda/src/main/java/com/myorg/S3ObjectLambdaTransformer.java b/java/s3-object-lambda/lambda/src/main/java/com/myorg/S3ObjectLambdaTransformer.java index 40109e9cb4..26b0398898 100644 --- a/java/s3-object-lambda/lambda/src/main/java/com/myorg/S3ObjectLambdaTransformer.java +++ b/java/s3-object-lambda/lambda/src/main/java/com/myorg/S3ObjectLambdaTransformer.java @@ -34,7 +34,7 @@ public class S3ObjectLambdaTransformer { * This method retrieves the original object from S3, applies transformations, * and returns the modified object data. * - * @param event The S3 Object Lambda event containing request details + * @param event The S3 Object Lambda event containing request details * @param context The Lambda execution context */ public void handleRequest(S3ObjectLambdaEvent event, Context context) { @@ -46,7 +46,7 @@ public void handleRequest(S3ObjectLambdaEvent event, Context context) { // Extract the pre-signed URL from the event context var objectContext = event.getGetObjectContext(); var s3Url = objectContext.getInputS3Url(); - + // Create HTTP client and fetch the original object var uri = URI.create(s3Url); var httpClient = HttpClient.newBuilder().build(); @@ -55,35 +55,40 @@ public void handleRequest(S3ObjectLambdaEvent event, Context context) { var requestBody = RequestBody.empty(); var responseBodyBytes = readBytes(response); var writeGetObjectResponseRequestBuilder = WriteGetObjectResponseRequest.builder() - .requestRoute(event.outputRoute()) - .requestToken(event.outputToken()) - .statusCode(response.statusCode()); + .requestRoute(event.outputRoute()) + .requestToken(event.outputToken()) + .statusCode(response.statusCode()); // Process successful responses (HTTP 200) if (response.statusCode() == 200) { // Build metadata object with content length and hash values var metadata = TransformedObject.Metadata.builder() - .withLength((long) responseBodyBytes.length) // Set content length - .withMD5(DigestUtils.md5Hex(responseBodyBytes)) // Calculate MD5 hash - .withSHA1(DigestUtils.sha1Hex(responseBodyBytes)) // Calculate SHA1 hash - .withSHA256(DigestUtils.sha256Hex(responseBodyBytes)) // Calculate SHA256 hash - .build(); + .withLength((long) responseBodyBytes.length) // Set content length + .withMD5(DigestUtils.md5Hex(responseBodyBytes)) // Calculate MD5 hash + .withSHA1(DigestUtils.sha1Hex(responseBodyBytes)) // Calculate SHA1 hash + .withSHA256(DigestUtils.sha256Hex(responseBodyBytes)) // Calculate SHA256 hash + .build(); // Create transformed object containing the metadata var transformedObject = TransformedObject.builder() - .withMetadata(metadata) - .build(); + .withMetadata(metadata) + .build(); // Log the transformed object for debugging log(context, "transformedObject: " + writeValue(objectMapper, transformedObject)); requestBody = RequestBody.fromString(writeValue(objectMapper, transformedObject)); - } else { - // Handle non-200 HTTP responses by setting the error message + } else { + // Handle non-200 HTTP responses by setting the error message writeGetObjectResponseRequestBuilder - .errorMessage(new String(responseBodyBytes)); // Convert error response body to string and set as error message + .errorMessage(new String(responseBodyBytes)); // Convert error response body to string and set as error + // message } - // Write the final response back to S3 Object Lambda with either transformed object or error details + // Write the final response back to S3 Object Lambda with either transformed + // object or error details s3Client.writeGetObjectResponse(writeGetObjectResponseRequestBuilder.build(), requestBody); } catch (IOException | InterruptedException e) { // Wrap and rethrow any IO or threading exceptions that occur during processing - throw new RuntimeException("Error while handling request: " + e.getMessage(), e) } + throw new RuntimeException("Error while handling request: " + e.getMessage(), e); + } + } + } /** From 316a43002003c5cf2b4571614efa2d8154fe1ffd Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Thu, 9 Jan 2025 11:16:41 -0600 Subject: [PATCH 4/4] Fix line ending --- .../src/main/java/com/myorg/S3ObjectLambdaTransformer.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/java/s3-object-lambda/lambda/src/main/java/com/myorg/S3ObjectLambdaTransformer.java b/java/s3-object-lambda/lambda/src/main/java/com/myorg/S3ObjectLambdaTransformer.java index 26b0398898..39ff452b74 100644 --- a/java/s3-object-lambda/lambda/src/main/java/com/myorg/S3ObjectLambdaTransformer.java +++ b/java/s3-object-lambda/lambda/src/main/java/com/myorg/S3ObjectLambdaTransformer.java @@ -89,8 +89,6 @@ public void handleRequest(S3ObjectLambdaEvent event, Context context) { } } - } - /** * Reads all bytes from the input stream of an HTTP response. * Converts the response stream into a byte array for processing.