diff --git a/java/s3-object-lambda/.gitignore b/java/s3-object-lambda/.gitignore new file mode 100644 index 0000000000..1db21f1629 --- /dev/null +++ b/java/s3-object-lambda/.gitignore @@ -0,0 +1,13 @@ +.classpath.txt +target +.classpath +.project +.idea +.settings +.vscode +*.iml + +# CDK asset staging directory +.cdk.staging +cdk.out + diff --git a/java/s3-object-lambda/README.md b/java/s3-object-lambda/README.md new file mode 100644 index 0000000000..4b8e8dee39 --- /dev/null +++ b/java/s3-object-lambda/README.md @@ -0,0 +1,75 @@ +# S3 Object Lambda + + + +--- +![Stability: Stable](https://img.shields.io/badge/stability-Stable-success.svg?style=for-the-badge) +> **This is a stable example. It should successfully build out of the box** +> +> This example is built on Construct Libraries marked "Stable" and does not have any infrastructure prerequisites to build. +--- + + + +## Overview + +This is a Java CDK example to create a [S3 Object Lambda](https://docs.aws.amazon.com/AmazonS3/latest/userguide/transforming-objects.html). +The main cloud infrastructure resources created with this example are: +- an S3 Bucket; +- a S3 Access Point; +- an S3 Object Lambda Access Point; +- a Lambda Function to process the GET object requests. + +Once this example is deployed, any object uploaded in the bucket created will be available through the S3 Object Lambda Access Point. +When trying to access an object through the S3 Object Lambda Access Point, a metadata output generated by the lambda function will be returned. +The object metadata output generated by the lambda is similar to the following: +```json +{ + "metadata": { + "length": 1048576, + "md5": "b6d81b360a5672d80c27430f39153e2c", + "sha1": "3b71f43ff30f4b15b5cd85dd9e95ebc7e84eb5a3", + "sha256": "30e14955ebf1352266dc2ff8067e68104607e750abb9d3b36582b8af909fcb58" + } +} +``` + +## Build + +**To build this example, you need to be in this example's [`infra`](infra) directory.** Then run the following: +```bash + npm install -g aws-cdk + npm install + cdk synth +``` +> [!NOTE] +> In order to build this project you need [Docker](https://www.docker.com/) (and [Docker Desktop](https://www.docker.com/products/docker-desktop/) if you want to use a GUI) installed. +> Moreover, the Docker daemon must be running while you execute the build or deploy commands. +> Unless the Docker daemon is running the Maven build of the lambda function ([`S3ObjectLambdaTransformer`](lambda/src/main/java/com/myorg/S3ObjectLambdaTransformer.java)) will fail. +> This is required because the CDK will build the code for the lambda function locally in a Docker container. +> You should also create a **virtual file share** for the local `/tmp` directory (see [Synchronized file shares](https://docs.docker.com/desktop/synchronized-file-sharing/)). +> This can be done from Docker Desktop (`Settings>Resources>File sharing`). + +> [!NOTE] +> When you build this project for the first time, the required Docker image ([sam/build-java17](https://gallery.ecr.aws/sam/build-java17)) will be pulled from AWS ECR. +> This will cause the first build to be slower. +> That is unless the required container image is already present on the system. + +## Deploy + +To deploy this example, you need to be in this example's [`infra`](infra) directory and run `cdk deploy`. +This will deploy / redeploy the Stack to AWS. +After the CDK deployment is successful, the URL of the S3 Object Lambda Access Point (the output named `S3ObjectLambdaStack.s3ObjectLambdaAccessPointUrl`) will be available in the terminal console. +At this point, this URL can be used for testing. +Copy the URL and paste it in the address bar of a browser. +This will take you to the access point in the AWS management console (you need to be authenticated for that). +From here you can access an S3 object to get the corresponding metadata (you need to have objects in the bucket for that). + +## Useful commands + + * `mvn package` compile and run tests + * `cdk ls` list all stacks in the app + * `cdk synth` emits the synthesized CloudFormation template + * `cdk deploy` deploy this stack to your default AWS account/region + * `cdk diff` compare deployed stack with current state + * `cdk docs` open CDK documentation diff --git a/java/s3-object-lambda/infra/.gitignore b/java/s3-object-lambda/infra/.gitignore new file mode 100644 index 0000000000..1db21f1629 --- /dev/null +++ b/java/s3-object-lambda/infra/.gitignore @@ -0,0 +1,13 @@ +.classpath.txt +target +.classpath +.project +.idea +.settings +.vscode +*.iml + +# CDK asset staging directory +.cdk.staging +cdk.out + diff --git a/java/s3-object-lambda/infra/cdk.json b/java/s3-object-lambda/infra/cdk.json new file mode 100644 index 0000000000..9f9fb58cd5 --- /dev/null +++ b/java/s3-object-lambda/infra/cdk.json @@ -0,0 +1,76 @@ +{ + "app": "mvn -e -q compile exec:java", + "watch": { + "include": [ + "**" + ], + "exclude": [ + "README.md", + "cdk*.json", + "target", + "pom.xml", + "src/test" + ] + }, + "context": { + "@aws-cdk/aws-lambda:recognizeLayerVersion": true, + "@aws-cdk/core:checkSecretUsage": true, + "@aws-cdk/core:target-partitions": [ + "aws", + "aws-cn" + ], + "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, + "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, + "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, + "@aws-cdk/aws-iam:minimizePolicies": true, + "@aws-cdk/core:validateSnapshotRemovalPolicy": true, + "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, + "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, + "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, + "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, + "@aws-cdk/core:enablePartitionLiterals": true, + "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true, + "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true, + "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true, + "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true, + "@aws-cdk/aws-route53-patters:useCertificate": true, + "@aws-cdk/customresources:installLatestAwsSdkDefault": false, + "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true, + "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true, + "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true, + "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true, + "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true, + "@aws-cdk/aws-redshift:columnId": true, + "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true, + "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true, + "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true, + "@aws-cdk/aws-kms:aliasNameRef": true, + "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true, + "@aws-cdk/core:includePrefixInUniqueNameGeneration": true, + "@aws-cdk/aws-efs:denyAnonymousAccess": true, + "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true, + "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true, + "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true, + "@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true, + "@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true, + "@aws-cdk/aws-rds:preventRenderingDeprecatedCredentials": true, + "@aws-cdk/aws-codepipeline-actions:useNewDefaultBranchForCodeCommitSource": true, + "@aws-cdk/aws-cloudwatch-actions:changeLambdaPermissionLogicalIdForLambdaAction": true, + "@aws-cdk/aws-codepipeline:crossAccountKeysDefaultValueToFalse": true, + "@aws-cdk/aws-codepipeline:defaultPipelineTypeToV2": true, + "@aws-cdk/aws-kms:reduceCrossAccountRegionPolicyScope": true, + "@aws-cdk/aws-eks:nodegroupNameAttribute": true, + "@aws-cdk/aws-ec2:ebsDefaultGp3Volume": true, + "@aws-cdk/aws-ecs:removeDefaultDeploymentAlarm": true, + "@aws-cdk/custom-resources:logApiResponseDataPropertyTrueDefault": false, + "@aws-cdk/aws-s3:keepNotificationInImportedBucket": false, + "@aws-cdk/aws-ecs:reduceEc2FargateCloudWatchPermissions": true, + "@aws-cdk/aws-dynamodb:resourcePolicyPerReplica": true, + "@aws-cdk/aws-ec2:ec2SumTImeoutEnabled": true, + "@aws-cdk/aws-appsync:appSyncGraphQLAPIScopeLambdaPermission": true, + "@aws-cdk/aws-rds:setCorrectValueForDatabaseInstanceReadReplicaInstanceResourceId": true, + "@aws-cdk/core:cfnIncludeRejectComplexResourceUpdateCreatePolicyIntrinsics": true, + "@aws-cdk/aws-lambda-nodejs:sdkV3ExcludeSmithyPackages": true, + "@aws-cdk/aws-stepfunctions-tasks:fixRunEcsTaskPolicy": true + } +} diff --git a/java/s3-object-lambda/infra/pom.xml b/java/s3-object-lambda/infra/pom.xml new file mode 100644 index 0000000000..5efd178559 --- /dev/null +++ b/java/s3-object-lambda/infra/pom.xml @@ -0,0 +1,73 @@ + + + 4.0.0 + + com.myorg.s3-object-lambda + infra + infra + 0.1 + + + UTF-8 + 17 + 17 + 2.171.1 + [10.0.0,11.0.0) + 5.7.1 + 1.18.34 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.11.0 + + ${java.version} + ${java.version} + ${java.version} + + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + com.myorg.S3ObjectLambdaApp + + + + + + + + + software.amazon.awscdk + aws-cdk-lib + ${cdk.version} + + + + software.constructs + constructs + ${constructs.version} + + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + + org.projectlombok + lombok + ${lombok.version} + provided + + + diff --git a/java/s3-object-lambda/infra/src/main/java/com/myorg/S3ObjectLambdaApp.java b/java/s3-object-lambda/infra/src/main/java/com/myorg/S3ObjectLambdaApp.java new file mode 100644 index 0000000000..9dbe76a49c --- /dev/null +++ b/java/s3-object-lambda/infra/src/main/java/com/myorg/S3ObjectLambdaApp.java @@ -0,0 +1,26 @@ +package com.myorg; + +import software.amazon.awscdk.App; +import software.amazon.awscdk.StackProps; + +/** + * Main CDK application class that serves as the entry point for deploying the S3 Object Lambda infrastructure. + */ +public class S3ObjectLambdaApp extends App { + + public static void main(final String... args) { + var app = new S3ObjectLambdaApp(); + var stackProps = StackProps.builder().build(); + app.createStack(stackProps); + app.synth(); + } + + /** + * Creates a new instance of the S3ObjectLambdaStack with the specified properties. + */ + public S3ObjectLambdaStack createStack(StackProps stackProps) { + return new S3ObjectLambdaStack(this, "S3ObjectLambdaStack", stackProps); + } + +} + diff --git a/java/s3-object-lambda/infra/src/main/java/com/myorg/S3ObjectLambdaStack.java b/java/s3-object-lambda/infra/src/main/java/com/myorg/S3ObjectLambdaStack.java new file mode 100644 index 0000000000..eb2a9c1a8d --- /dev/null +++ b/java/s3-object-lambda/infra/src/main/java/com/myorg/S3ObjectLambdaStack.java @@ -0,0 +1,192 @@ +package com.myorg; + +import software.amazon.awscdk.*; +import software.amazon.awscdk.services.iam.*; +import software.amazon.awscdk.services.lambda.Code; +import software.amazon.awscdk.services.lambda.Function; +import software.amazon.awscdk.services.lambda.Permission; +import software.amazon.awscdk.services.lambda.Runtime; +import software.amazon.awscdk.services.s3.BlockPublicAccess; +import software.amazon.awscdk.services.s3.Bucket; +import software.amazon.awscdk.services.s3.BucketAccessControl; +import software.amazon.awscdk.services.s3.BucketEncryption; +import software.amazon.awscdk.services.s3.assets.AssetOptions; +import software.constructs.Construct; +import software.amazon.awscdk.services.s3objectlambda.CfnAccessPoint; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static java.util.Collections.singletonList; +import static software.amazon.awscdk.BundlingOutput.ARCHIVED; +import static software.amazon.awscdk.services.s3objectlambda.CfnAccessPoint.*; + +public class S3ObjectLambdaStack extends Stack { + private static final String S3_ACCESS_POINT_NAME = "s3-access-point"; + private static final String OBJECT_LAMBDA_ACCESS_POINT_NAME = "object-lambda-access-point"; + + /** + * Constructs a new S3ObjectLambdaStack. + */ + public S3ObjectLambdaStack(final Construct scope, final String id, final StackProps props) { + super(scope, id, props); + + // Construct the access point ARN using the region, account ID and access point name + var accessPoint = "arn:aws:s3:" + Aws.REGION + ":" + Aws.ACCOUNT_ID + ":accesspoint/" + S3_ACCESS_POINT_NAME; + + // Create a new S3 bucket with secure configuration including: + var s3ObjectLambdaBucket = Bucket.Builder.create(this, "S3ObjectLambdaBucket") + .removalPolicy(RemovalPolicy.RETAIN) + .autoDeleteObjects(false) + .accessControl(BucketAccessControl.BUCKET_OWNER_FULL_CONTROL) + .encryption(BucketEncryption.S3_MANAGED) + .blockPublicAccess(BlockPublicAccess.BLOCK_ALL) + .build(); + + // Create bucket policy statement allowing access through access points + var s3ObjectLambdaBucketPolicyStatement = PolicyStatement.Builder.create() + .actions(List.of("*")) + .principals(List.of(new AnyPrincipal())) + .resources(List.of( + s3ObjectLambdaBucket.getBucketArn(), + s3ObjectLambdaBucket.arnForObjects("*") + )) + .conditions( + Map.of( + "StringEquals", Map.of( + "s3:DataAccessPointAccount", Aws.ACCOUNT_ID + ) + ) + ) + .build(); + + // Attach the policy to the bucket + s3ObjectLambdaBucket.addToResourcePolicy(s3ObjectLambdaBucketPolicyStatement); + + // Create the Lambda function that will transform objects + var s3ObjectLambdaFunction = createS3ObjectLambdaFunction(); + + // Add permission for Lambda to write GetObject responses for s3 object + var s3ObjectLambdaFunctionPolicyStatement = PolicyStatement.Builder.create() + .effect(Effect.ALLOW) + .resources(List.of("*")) + .actions(List.of("s3-object-lambda:WriteGetObjectResponse")) + .build(); + s3ObjectLambdaFunction.addToRolePolicy(s3ObjectLambdaFunctionPolicyStatement); + + // Add permission for the account root to invoke the Lambda function + var s3ObjectLambdaFunctionPermission = Permission.builder() + .action("lambda:InvokeFunction") + .principal(new AccountRootPrincipal()) + .sourceAccount(Aws.ACCOUNT_ID) + .build(); + s3ObjectLambdaFunction.addPermission("S3ObjectLambdaPermission", s3ObjectLambdaFunctionPermission); + + // Create policy allowing Lambda function to get objects through the access point + var s3ObjectLambdaAccessPointPolicyStatement = PolicyStatement.Builder.create() + .sid("S3ObjectLambdaAccessPointPolicyStatement") + .effect(Effect.ALLOW) + .actions(List.of("s3:GetObject")) + .principals(List.of( + new ArnPrincipal(Objects.requireNonNull(s3ObjectLambdaFunction.getRole()).getRoleArn()) + ) + ) + .resources(List.of(accessPoint + "/object/*")) + .build(); + + // Create policy document containing the access point policy + var s3ObjectLambdaAccessPointPolicyDocument = PolicyDocument.Builder.create() + .statements(List.of( + s3ObjectLambdaAccessPointPolicyStatement + )) + .build(); + + // Create the S3 access point for direct bucket access + software.amazon.awscdk.services.s3.CfnAccessPoint.Builder.create(this, "S3ObjectLambdaS3AccessPoint") + .bucket(s3ObjectLambdaBucket.getBucketName()) + .name(S3_ACCESS_POINT_NAME) + .policy(s3ObjectLambdaAccessPointPolicyDocument) + .build(); + + // Create the Object Lambda access point that will transform objects + var s3ObjectLambdaAccessPoint = CfnAccessPoint.Builder.create(this, "S3ObjectLambdaAccessPoint") + .name(OBJECT_LAMBDA_ACCESS_POINT_NAME) + .objectLambdaConfiguration(ObjectLambdaConfigurationProperty.builder() + .supportingAccessPoint(accessPoint) + .transformationConfigurations(List.of( + TransformationConfigurationProperty.builder() + .actions(List.of("GetObject")) + .contentTransformation( + Map.of( + "AwsLambda", Map.of( + "FunctionArn", s3ObjectLambdaFunction.getFunctionArn() + ) + ) + ) + .build() + ) + ) + .build() + ) + .build(); + CfnOutput.Builder.create(this, "s3ObjectLambdaBucketArn") + .value(s3ObjectLambdaBucket.getBucketArn()) // Export bucket ARN + .build(); + CfnOutput.Builder.create(this, "s3ObjectLambdaFunctionArn") + .value(s3ObjectLambdaFunction.getFunctionArn()) // Export Lambda function ARN + .build(); + CfnOutput.Builder.create(this, "s3ObjectLambdaAccessPointArn") + .value(s3ObjectLambdaAccessPoint.getAttrArn()) // Export access point ARN + .build(); + + // Create output with Console URL for easy access to the Lambda access point + CfnOutput.Builder.create(this, "s3ObjectLambdaAccessPointUrl") + .value("https://console.aws.amazon.com/s3/olap/" + Aws.ACCOUNT_ID + "/" + OBJECT_LAMBDA_ACCESS_POINT_NAME + "?region=" + Aws.REGION) + .build(); + } + + /** + * Creates the Lambda function that will process S3 Object Lambda requests. + * This method configures the function's runtime, code, and build process. + * + * @return A Lambda Function construct configured for S3 Object Lambda processing + */ + private Function createS3ObjectLambdaFunction() { + // Define Maven packaging commands to build the Lambda function + List packagingInstructions = List.of( + "/bin/sh", + "-c", + // Build the project and copy the JAR to the asset output directory + "mvn -e -q clean package && cp /asset-input/target/lambda-1.0-SNAPSHOT.jar /asset-output/" + ); + // Configure the bundling options for packaging the Lambda function + var builderOptions = BundlingOptions.builder() + .command(packagingInstructions) // Set the Maven build commands + .image(Runtime.JAVA_17.getBundlingImage()) // Use Java 17 runtime image + .volumes( + singletonList( + DockerVolume.builder() + .hostPath(System.getProperty("user.home") + "/.m2/") + .containerPath("/root/.m2/") + .build() + )) + .user("root") + .outputType(ARCHIVED) + .build(); + + // Create the Lambda function with specified configuration + return Function.Builder.create(this, "S3ObjectLambdaFunction") + .runtime(Runtime.JAVA_17) // Set Java 17 runtime + .functionName("S3ObjectLambdaFunction") // Set function name + .memorySize(2048) // Allocate 2GB memory + .code( + Code.fromAsset( + "../lambda/", + AssetOptions.builder().bundling(builderOptions).build() + ) + ) + .handler("com.myorg.S3ObjectLambdaTransformer::handleRequest") + .build(); + } +} \ No newline at end of file diff --git a/java/s3-object-lambda/infra/src/test/java/com/myorg/S3ObjectLambdaAppTest.java b/java/s3-object-lambda/infra/src/test/java/com/myorg/S3ObjectLambdaAppTest.java new file mode 100644 index 0000000000..a84abe1aa1 --- /dev/null +++ b/java/s3-object-lambda/infra/src/test/java/com/myorg/S3ObjectLambdaAppTest.java @@ -0,0 +1,36 @@ +package com.myorg; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import software.amazon.awscdk.StackProps; +import software.amazon.awscdk.assertions.Template; + +import static com.myorg.TestUtils.TestConstants.*; +import static com.myorg.TestUtils.isResourceInStack; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class S3ObjectLambdaAppTest { + + private static S3ObjectLambdaApp s3ObjectLambdaApp; + + @BeforeAll + public static void setUp() { + s3ObjectLambdaApp = new S3ObjectLambdaApp(); + } + + @Test + public void testCreateStack() { + var stackProps = StackProps.builder() + .build(); + var stack = s3ObjectLambdaApp.createStack(stackProps); + var template = Template.fromStack(stack); + assertTrue(isResourceInStack(BUCKET_MATCH_PAIR, template), "The expected S3 bucket is not present in the resources of the stack."); + assertTrue(isResourceInStack(BUCKET_POLICY_MATCH_PAIR, template), "The expected S3 bucket policy is not present in the resources of the stack."); + assertTrue(isResourceInStack(IAM_ROLE_MATCH_PAIR, template), "The expected IAM role for the lambda function is not present in the resources of the stack."); + assertTrue(isResourceInStack(IAM_POLICY_MATCH_PAIR, template), "The expected IAM policy for the lambda function is not present in the resources of the stack."); + assertTrue(isResourceInStack(LAMBDA_FUNCTION_MATCH_PAIR, template), "The expected lambda function is not present in the resources of the stack."); + assertTrue(isResourceInStack(LAMBDA_PERMISSION_MATCH_PAIR, template), "The expected lambda permission is not present in the resources of the stack."); + assertTrue(isResourceInStack(S3_ACCESS_POINT_MATCH_PAIR, template), "The expected S3 access point is not present in the resources of the stack."); + assertTrue(isResourceInStack(S3_OBJECT_LAMBDA_ACCESS_POINT, template), "The expected S3 object lambda access point is not present in the resources of the stack."); + } +} diff --git a/java/s3-object-lambda/infra/src/test/java/com/myorg/TestUtils.java b/java/s3-object-lambda/infra/src/test/java/com/myorg/TestUtils.java new file mode 100644 index 0000000000..bec15a7c00 --- /dev/null +++ b/java/s3-object-lambda/infra/src/test/java/com/myorg/TestUtils.java @@ -0,0 +1,247 @@ +package com.myorg; + +import lombok.Builder; +import org.junit.platform.commons.util.StringUtils; +import software.amazon.awscdk.assertions.Template; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Stream; + +public class TestUtils { + + @Builder(builderClassName = "MatchPairBuilder", setterPrefix = "with") + public record MatchPair(String type, Object props) { + + } + + public static class TestConstants { + public static final MatchPair BUCKET_MATCH_PAIR = MatchPair.builder() + .withType("AWS::S3::Bucket") + .withProps( + Map.of( + "UpdateReplacePolicy", "Retain", + "DeletionPolicy", "Retain", + "Properties", Map.of( + "AccessControl", "BucketOwnerFullControl", + "BucketEncryption", Map.of( + "ServerSideEncryptionConfiguration", List.of( + Map.of( + "ServerSideEncryptionByDefault", Map.of( + "SSEAlgorithm", "AES256" + ) + ) + ) + ), + "PublicAccessBlockConfiguration", Map.of( + "BlockPublicAcls", true, + "BlockPublicPolicy", true, + "IgnorePublicAcls", true, + "RestrictPublicBuckets", true + ) + ) + ) + ) + .build(); + public static final MatchPair BUCKET_POLICY_MATCH_PAIR = MatchPair.builder() + .withType("AWS::S3::BucketPolicy") + .withProps( + Map.of( + "Properties", Map.of( + "PolicyDocument", Map.of( + "Statement", List.of( + Map.of( + "Action", "*", + "Condition", Map.of( + "StringEquals", Map.of( + "s3:DataAccessPointAccount", Map.of( + "Ref", "AWS::AccountId" + ) + ) + ), + "Effect", "Allow", + "Principal", Map.of( + "AWS", "*" + ) + ) + ), + "Version", "2012-10-17" + ) + ) + ) + ) + .build(); + public static final MatchPair IAM_ROLE_MATCH_PAIR = MatchPair.builder() + .withType("AWS::IAM::Role") + .withProps( + Map.of( + "Properties", Map.of( + "AssumeRolePolicyDocument", Map.of( + "Statement", List.of( + Map.of( + "Action", "sts:AssumeRole", + "Effect", "Allow", + "Principal", Map.of( + "Service", "lambda.amazonaws.com" + ) + ) + ), + "Version", "2012-10-17" + ), + "ManagedPolicyArns", List.of( + Map.of( + "Fn::Join", List.of( + "", List.of( + "arn:", + Map.of( + "Ref", "AWS::Partition" + ), + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ) + ) + ) + ) + ) + ) + ) + .build(); + public static final MatchPair IAM_POLICY_MATCH_PAIR = MatchPair.builder() + .withType("AWS::IAM::Policy") + .withProps( + Map.of( + "Properties", Map.of( + "PolicyDocument", Map.of( + "Statement", List.of( + Map.of( + "Action", "s3-object-lambda:WriteGetObjectResponse", + "Effect", "Allow", + "Resource", "*" + ) + ), + "Version", "2012-10-17" + ) + ) + ) + ) + .build(); + public static final MatchPair LAMBDA_FUNCTION_MATCH_PAIR = MatchPair.builder() + .withType("AWS::Lambda::Function") + .withProps( + Map.of( + "Properties", Map.of( + "FunctionName", "S3ObjectLambdaFunction", + "Handler", "com.myorg.S3ObjectLambdaTransformer::handleRequest", + "MemorySize", 2048, + "Runtime", "java17" + ) + ) + ) + .build(); + public static final MatchPair LAMBDA_PERMISSION_MATCH_PAIR = MatchPair.builder() + .withType("AWS::Lambda::Permission") + .withProps( + Map.of( + "Properties", Map.of( + "Action", "lambda:InvokeFunction", + "Principal", Map.of( + "Ref", "AWS::AccountId" + ), + "SourceAccount", Map.of( + "Ref", "AWS::AccountId" + ) + ) + ) + ) + .build(); + public static final MatchPair S3_ACCESS_POINT_MATCH_PAIR = MatchPair.builder() + .withType("AWS::S3::AccessPoint") + .withProps( + Map.of( + "Properties", Map.of( + "Name", "s3-access-point", + "Policy", Map.of( + "Statement", List.of( + Map.of( + "Action", "s3:GetObject", + "Effect", "Allow", + "Resource", Map.of( + "Fn::Join", List.of( + "", + List.of( + "arn:aws:s3:", + Map.of( + "Ref", "AWS::Region" + ), + ":", + Map.of( + "Ref", "AWS::AccountId" + ), + ":accesspoint/s3-access-point/object/*" + ) + ) + ), + "Sid", "S3ObjectLambdaAccessPointPolicyStatement" + ) + ), + "Version", "2012-10-17" + ) + ) + ) + ) + .build(); + public static final MatchPair S3_OBJECT_LAMBDA_ACCESS_POINT = MatchPair.builder() + .withType("AWS::S3ObjectLambda::AccessPoint") + .withProps( + Map.of( + "Properties", Map.of( + "Name", "object-lambda-access-point", + "ObjectLambdaConfiguration", Map.of( + "SupportingAccessPoint", Map.of( + "Fn::Join", List.of( + "", + List.of( + "arn:aws:s3:", + Map.of( + "Ref", "AWS::Region" + ), + ":", + Map.of( + "Ref", "AWS::AccountId" + ), + ":accesspoint/s3-access-point" + ) + ) + ), + "TransformationConfigurations", List.of( + Map.of( + "Actions", List.of( + "GetObject" + ), + "ContentTransformation", Map.of( + "AwsLambda", Map.of() + ) + ) + ) + ) + ) + ) + ) + .build(); + } + + public static boolean isResourceInStack(MatchPair matchPair, Template template) { + return Optional.ofNullable(matchPair) + .filter(pair -> StringUtils.isNotBlank(pair.type)) + .filter(pair -> pair.props != null) + .map(pair -> template.findResources(pair.type, pair.props)) + .map(Map::entrySet) + .map(Set::stream) + .map(Stream::count) + .filter(count -> count == 1) + .isPresent(); + } + + +} diff --git a/java/s3-object-lambda/lambda/.gitignore b/java/s3-object-lambda/lambda/.gitignore new file mode 100644 index 0000000000..1db21f1629 --- /dev/null +++ b/java/s3-object-lambda/lambda/.gitignore @@ -0,0 +1,13 @@ +.classpath.txt +target +.classpath +.project +.idea +.settings +.vscode +*.iml + +# CDK asset staging directory +.cdk.staging +cdk.out + diff --git a/java/s3-object-lambda/lambda/pom.xml b/java/s3-object-lambda/lambda/pom.xml new file mode 100644 index 0000000000..0028d4a954 --- /dev/null +++ b/java/s3-object-lambda/lambda/pom.xml @@ -0,0 +1,119 @@ + + + 4.0.0 + + com.myorg.s3-object-lambda + lambda + lambda + 1.0-SNAPSHOT + + + UTF-8 + 17 + 17 + 1.2.3 + 3.11.0 + 2.29.43 + 1.17.1 + 2.18.0 + 1.18.34 + 5.7.1 + 5.14.2 + + + + + com.amazonaws + aws-lambda-java-core + ${aws-lambda-java-core.version} + + + com.amazonaws + aws-lambda-java-events + ${aws-lambda-java-events.version} + + + software.amazon.awssdk + s3 + ${awssdk-s3} + + + commons-codec + commons-codec + ${commons-codec} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson-databind.version} + + + org.projectlombok + lombok + ${lombok.version} + provided + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + org.mockito + mockito-core + ${mockito.version} + test + + + org.mockito + mockito-junit-jupiter + ${mockito.version} + test + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.11.0 + + ${java.version} + ${java.version} + ${java.version} + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.1 + + + -javaagent:${settings.localRepository}/org/mockito/mockito-core/${mockito.version}/mockito-core-${mockito.version}.jar + -Xshare:off + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.0 + + false + + + + package + + shade + + + + + + + + diff --git a/java/s3-object-lambda/lambda/src/main/java/com/myorg/S3ObjectLambdaTransformer.java b/java/s3-object-lambda/lambda/src/main/java/com/myorg/S3ObjectLambdaTransformer.java new file mode 100644 index 0000000000..39ff452b74 --- /dev/null +++ b/java/s3-object-lambda/lambda/src/main/java/com/myorg/S3ObjectLambdaTransformer.java @@ -0,0 +1,149 @@ +package com.myorg; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.events.S3ObjectLambdaEvent; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.myorg.model.TransformedObject; +import org.apache.commons.codec.digest.DigestUtils; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.WriteGetObjectResponseRequest; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.Optional; + +import static java.net.http.HttpResponse.BodyHandlers.ofInputStream; + +/** + * AWS Lambda function handler that processes S3 Object Lambda requests. + * This class implements the transformation logic for S3 objects accessed + * through + * the Object Lambda Access Point. + */ +public class S3ObjectLambdaTransformer { + + /** + * Main handler method that processes S3 Object Lambda events. + * This method retrieves the original object from S3, applies transformations, + * and returns the modified object data. + * + * @param event The S3 Object Lambda event containing request details + * @param context The Lambda execution context + */ + public void handleRequest(S3ObjectLambdaEvent event, Context context) { + try (var s3Client = S3Client.create()) { + // Create JSON mapper and log the incoming event + var objectMapper = createObjectMapper(); + log(context, "event: " + writeValue(objectMapper, event)); + + // Extract the pre-signed URL from the event context + var objectContext = event.getGetObjectContext(); + var s3Url = objectContext.getInputS3Url(); + + // Create HTTP client and fetch the original object + var uri = URI.create(s3Url); + var httpClient = HttpClient.newBuilder().build(); + var httpRequest = HttpRequest.newBuilder(uri).GET().build(); + var response = httpClient.send(httpRequest, ofInputStream()); + var requestBody = RequestBody.empty(); + var responseBodyBytes = readBytes(response); + var writeGetObjectResponseRequestBuilder = WriteGetObjectResponseRequest.builder() + .requestRoute(event.outputRoute()) + .requestToken(event.outputToken()) + .statusCode(response.statusCode()); + // Process successful responses (HTTP 200) + if (response.statusCode() == 200) { + // Build metadata object with content length and hash values + var metadata = TransformedObject.Metadata.builder() + .withLength((long) responseBodyBytes.length) // Set content length + .withMD5(DigestUtils.md5Hex(responseBodyBytes)) // Calculate MD5 hash + .withSHA1(DigestUtils.sha1Hex(responseBodyBytes)) // Calculate SHA1 hash + .withSHA256(DigestUtils.sha256Hex(responseBodyBytes)) // Calculate SHA256 hash + .build(); + // Create transformed object containing the metadata + var transformedObject = TransformedObject.builder() + .withMetadata(metadata) + .build(); + // Log the transformed object for debugging + log(context, "transformedObject: " + writeValue(objectMapper, transformedObject)); + requestBody = RequestBody.fromString(writeValue(objectMapper, transformedObject)); + } else { + // Handle non-200 HTTP responses by setting the error message + writeGetObjectResponseRequestBuilder + .errorMessage(new String(responseBodyBytes)); // Convert error response body to string and set as error + // message + } + // Write the final response back to S3 Object Lambda with either transformed + // object or error details + s3Client.writeGetObjectResponse(writeGetObjectResponseRequestBuilder.build(), requestBody); + } catch (IOException | InterruptedException e) { + // Wrap and rethrow any IO or threading exceptions that occur during processing + throw new RuntimeException("Error while handling request: " + e.getMessage(), e); + } + } + + /** + * Reads all bytes from the input stream of an HTTP response. + * Converts the response stream into a byte array for processing. + * + * @param response HTTP response containing the input stream to read + * @return byte array containing all the data from the input stream + * @throws IOException if an I/O error occurs while reading the stream + */ + private static byte[] readBytes(HttpResponse response) throws IOException { + try (var inputStream = response.body()) { + return inputStream.readAllBytes(); + } + } + + /** + * Logs a message to CloudWatch using the Lambda context logger. + * Prefixes the message with the request ID for tracing purposes. + * + * @param context Lambda execution context containing the logger + * @param message Message to be logged + */ + private static void log(Context context, String message) { + if (message != null) { + Optional.ofNullable(context) + .map(Context::getLogger) + .ifPresent(lambdaLogger -> lambdaLogger.log(message)); + } + } + + /** + * Creates and configures a Jackson ObjectMapper for JSON serialization. + * Enables pretty printing and disables failing on empty beans. + * + * @return Configured ObjectMapper instance + */ + private static ObjectMapper createObjectMapper() { + var objectMapper = new ObjectMapper(); + objectMapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + return objectMapper; + } + + /** + * Serializes an object to a JSON string using the provided ObjectMapper. + * Handles JsonProcessingException by returning an error message. + * + * @param objectMapper The ObjectMapper to use for serialization + * @param object The object to serialize + * @return JSON string representation of the object or error message + */ + private static String writeValue(ObjectMapper objectMapper, Object object) { + try { + return objectMapper.writeValueAsString(object); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/java/s3-object-lambda/lambda/src/main/java/com/myorg/model/TransformedObject.java b/java/s3-object-lambda/lambda/src/main/java/com/myorg/model/TransformedObject.java new file mode 100644 index 0000000000..fc6019cdab --- /dev/null +++ b/java/s3-object-lambda/lambda/src/main/java/com/myorg/model/TransformedObject.java @@ -0,0 +1,24 @@ +package com.myorg.model; + +import lombok.*; + +/** + * Represents a transformed S3 object with its metadata. + * This class uses Lombok annotations for builder pattern, toString, and data methods generation. + */ +@Builder(setterPrefix = "with", builderClassName = "TransformedObjectBuilder") +@ToString +@Data +public class TransformedObject { + + /** + * Record class representing the metadata of a transformed object. + * Contains various hash values and the object length for verification purposes. + */ + @Builder(setterPrefix = "with", builderClassName = "MetadataBuilder") + public record Metadata(Long length, String MD5, String SHA1, String SHA256) { + + } + + private Metadata metadata; +} diff --git a/java/s3-object-lambda/lambda/src/test/java/com/myorg/S3ObjectLambdaTransformerTest.java b/java/s3-object-lambda/lambda/src/test/java/com/myorg/S3ObjectLambdaTransformerTest.java new file mode 100644 index 0000000000..2053bc3e5a --- /dev/null +++ b/java/s3-object-lambda/lambda/src/test/java/com/myorg/S3ObjectLambdaTransformerTest.java @@ -0,0 +1,65 @@ +package com.myorg; + +import com.amazonaws.services.lambda.runtime.events.S3ObjectLambdaEvent; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.junit.jupiter.MockitoExtension; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.WriteGetObjectResponseRequest; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; + +import static com.amazonaws.services.lambda.runtime.events.S3ObjectLambdaEvent.GetObjectContext; +import static java.net.http.HttpResponse.BodyHandlers.ofInputStream; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class S3ObjectLambdaTransformerTest { + + @Mock + private S3Client s3Client; + @Mock + private HttpClient.Builder httpClientBuilder; + @Mock + private HttpClient httpClient; + @Mock + private HttpResponse response; + + private final S3ObjectLambdaTransformer s3ObjectLambdaTransformer = new S3ObjectLambdaTransformer(); + + @Test + void testHandleRequest() throws IOException, InterruptedException { + var s3ObjectContent = "S3 test data."; + var event = S3ObjectLambdaEvent.builder() + .withGetObjectContext( + GetObjectContext.builder() + .withInputS3Url("https://s3-access-point-111111111111.s3-accesspoint.eu-north-1.amazonaws.com/test.txt") + .build() + ) + .build(); + try ( + MockedStatic mockedStaticS3Client = mockStatic(S3Client.class); + MockedStatic mockedStaticHttpClient = mockStatic(HttpClient.class) + ) { + mockedStaticS3Client.when(S3Client::create).thenReturn(s3Client); + mockedStaticHttpClient.when(HttpClient::newBuilder).thenReturn(httpClientBuilder); + when(httpClientBuilder.build()).thenReturn(httpClient); + when(httpClient.send(any(HttpRequest.class), eq(ofInputStream()))).thenReturn(response); + when(response.statusCode()).thenReturn(200); + when(response.body()).thenReturn(new ByteArrayInputStream(s3ObjectContent.getBytes(StandardCharsets.UTF_8))); + s3ObjectLambdaTransformer.handleRequest(event, null); + // test if the writeGetObjectResponse method of the s3Client instance is called to send the transformed object + verify(s3Client).writeGetObjectResponse(any(WriteGetObjectResponseRequest.class), any(RequestBody.class)); + } + } + +}