Merge pull request #12 from CodingNagger/feature/s3-deployment

S3 Deployment added for Layers and Lambdas

Author: Andrea-Scuderi

LambdaDeployerPolicyExample.json

@@ -49,6 +49,12 @@
                 "arn:aws:lambda:*:*:function:*",
                 "arn:aws:lambda:*:*:layer:*"
             ]
+        },
+        {
+            "Sid": "VisualEditor4",
+            "Effect": "Allow",
+            "Action": "s3:*",
+            "Resource": "arn:aws:s3:::aws-lambda-swift-sprinter*"
         }
     ]
 }

Makefile

@@ -30,12 +30,20 @@ LAMBDA_HANDLER?=$(SWIFT_EXECUTABLE).helloWorld
 # LAMBDA_FUNCTION_NAME=S3Test
 # LAMBDA_HANDLER=$(SWIFT_EXECUTABLE).getObject
 
+# AWS Configuration
+IAM_ROLE_NAME?=lambda_sprinter_basic_execution
+AWS_PROFILE?=default
+AWS_BUCKET?=aws-lambda-swift-sprinter
+
 # Internals
 LAMBDA_ZIP=lambda.zip
 SHARED_LIBS_FOLDER=swift-shared-libs
 LAYER_ZIP=swift-lambda-runtime-$(LAYER_VERSION).zip
-LAMBDA_BUILD_PATH=.build
-IAM_ROLE_NAME=lambda_sprinter_basic_execution
+ROOT_BUILD_PATH=./.build
+LAYER_BUILD_PATH=$(ROOT_BUILD_PATH)/layer
+LAMBDA_BUILD_PATH=$(ROOT_BUILD_PATH)/lambda
+TMP_BUILD_PATH=$(ROOT_BUILD_PATH)/tmp
+DATETIME=$(shell date +'%y%m%d-%H%M%S')
 
 # use this for local development
 MOUNT_ROOT=$(shell pwd)/..
@@ -45,10 +53,6 @@ DOCKER_PROJECT_PATH=aws-lambda-swift-sprinter/$(SWIFT_PROJECT_PATH)
 # MOUNT_ROOT=$(shell pwd)
 # DOCKER_PROJECT_PATH=$(SWIFT_PROJECT_PATH)
 
-# AWS Configuration
-AWS_PROFILE?=default
-AWS_BUCKET?=my-s3-bucket
-
 swift_test:
 	docker run \
 			--rm \
@@ -110,13 +114,18 @@ test_package:
 			swift test
 
 create_build_directory:
-	if [ ! -d "$(LAMBDA_BUILD_PATH)" ]; then mkdir $(LAMBDA_BUILD_PATH); fi
+	if [ ! -d "$(LAMBDA_BUILD_PATH)" ]; then mkdir -p $(LAMBDA_BUILD_PATH); fi
+	if [ ! -d "$(LAYER_BUILD_PATH)" ]; then mkdir -p $(LAYER_BUILD_PATH); fi
+	if [ ! -d "$(TMP_BUILD_PATH)" ]; then mkdir -p $(TMP_BUILD_PATH); fi
 
 package_lambda: clean_lambda create_build_directory build_lambda
 	zip -r -j $(LAMBDA_BUILD_PATH)/$(LAMBDA_ZIP) $(SWIFT_PROJECT_PATH)/.build/$(SWIFT_CONFIGURATION)/$(SWIFT_EXECUTABLE)
 
+clean_all: clean_layer clean_lambda
+	rm -r $(ROOT_BUILD_PATH)
+
 clean_layer:
-	rm $(LAMBDA_BUILD_PATH)/$(LAYER_ZIP) || true
+	rm $(LAYER_BUILD_PATH)/$(LAYER_ZIP) || true
 	rm -r $(SHARED_LIBS_FOLDER) || true
 
 package_layer: create_build_directory clean_layer
@@ -134,34 +143,56 @@ package_layer: create_build_directory clean_layer
 			--workdir "/src" \
 			$(SWIFT_DOCKER_IMAGE) \
 			cp -t $(SHARED_LIBS_FOLDER)/lib $(SHARED_LIBRARIES)
-	zip -r $(LAMBDA_BUILD_PATH)/$(LAYER_ZIP) bootstrap $(SHARED_LIBS_FOLDER)
+	zip -r $(LAYER_BUILD_PATH)/$(LAYER_ZIP) bootstrap $(SHARED_LIBS_FOLDER)
+
+upload_build_to_s3: create_lambda_s3_key
+	aws s3 sync --acl public-read "$(LAMBDA_BUILD_PATH)" s3://$(AWS_BUCKET)/$(LAMBDA_S3_UPLOAD_PATH) --profile $(AWS_PROFILE)
 
-upload_build_to_s3:
-	aws s3 sync --acl public-read ./.build s3://$(AWS_BUCKET)/ --profile $(AWS_PROFILE)
+create_layer_s3_key:
+	$(eval LAYER_S3_UPLOAD_PATH := layers/$(SWIFT_LAMBDA_LIBRARY))
+
+upload_lambda_layer_with_s3: create_layer_s3_key
+	aws s3 sync --acl public-read "$(LAYER_BUILD_PATH)" s3://$(AWS_BUCKET)/$(LAYER_S3_UPLOAD_PATH) --profile $(AWS_PROFILE)
+	aws lambda publish-layer-version --layer-name $(SWIFT_LAMBDA_LIBRARY) --description "AWS Custom Runtime Swift Shared Libraries with NIO" --content "S3Bucket=$(AWS_BUCKET),S3Key=$(LAYER_S3_UPLOAD_PATH)/$(LAYER_ZIP)" --output text --query LayerVersionArn --profile $(AWS_PROFILE) > $(TMP_BUILD_PATH)/$(SWIFT_LAMBDA_LIBRARY)-arn.txt
+	cat $(TMP_BUILD_PATH)/$(SWIFT_LAMBDA_LIBRARY)-arn.txt
 
 upload_lambda_layer:
-	aws lambda publish-layer-version --layer-name $(SWIFT_LAMBDA_LIBRARY) --description "AWS Custom Runtime Swift Shared Libraries with NIO" --zip-file fileb://$(LAMBDA_BUILD_PATH)/$(LAYER_ZIP) --output text --query LayerVersionArn --profile $(AWS_PROFILE) > $(LAMBDA_BUILD_PATH)/$(SWIFT_LAMBDA_LIBRARY)-arn.txt
-	cat $(LAMBDA_BUILD_PATH)/$(SWIFT_LAMBDA_LIBRARY)-arn.txt
+	aws lambda publish-layer-version --layer-name $(SWIFT_LAMBDA_LIBRARY) --description "AWS Custom Runtime Swift Shared Libraries with NIO" --zip-file fileb://$(LAYER_BUILD_PATH)/$(LAYER_ZIP) --output text --query LayerVersionArn --profile $(AWS_PROFILE) > $(TMP_BUILD_PATH)/$(SWIFT_LAMBDA_LIBRARY)-arn.txt
+	cat $(TMP_BUILD_PATH)/$(SWIFT_LAMBDA_LIBRARY)-arn.txt
 
 create_role:
 	aws iam create-role --role-name $(IAM_ROLE_NAME) --description "Allows Lambda functions to call AWS services on your behalf." --assume-role-policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["sts:AssumeRole"],"Principal":{"Service":["lambda.amazonaws.com"]}}]}' --profile $(AWS_PROFILE) || true
 	aws iam attach-role-policy --role-name $(IAM_ROLE_NAME) --policy-arn "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" --profile $(AWS_PROFILE) || true
     $(eval IAM_ROLE_ARN := $(shell aws iam list-roles --query "Roles[? RoleName == '$(IAM_ROLE_NAME)'].Arn" --output text --profile $(AWS_PROFILE)))
 
 create_lambda: create_role package_lambda
-	$(eval LAMBDA_LAYER_ARN := $(shell cat $(LAMBDA_BUILD_PATH)/$(SWIFT_LAMBDA_LIBRARY)-arn.txt))
+	$(eval LAMBDA_LAYER_ARN := $(shell cat $(TMP_BUILD_PATH)/$(SWIFT_LAMBDA_LIBRARY)-arn.txt))
+	$(info "$(LAMBDA_LAYER_ARN)")
+	aws lambda create-function --function-name $(LAMBDA_FUNCTION_NAME) --runtime provided --handler $(LAMBDA_HANDLER) --role "$(IAM_ROLE_ARN)" --zip-file fileb://$(LAMBDA_BUILD_PATH)/$(LAMBDA_ZIP) --layers "$(LAMBDA_LAYER_ARN)" --profile $(AWS_PROFILE)
+
+create_lambda_s3_key:
+	$(eval LAMBDA_S3_UPLOAD_PATH := lambdas/$(LAMBDA_FUNCTION_NAME)/$(DATETIME))
+
+create_lambda_with_s3: create_role package_lambda upload_build_to_s3
+	$(eval LAMBDA_LAYER_ARN := $(shell cat $(TMP_BUILD_PATH)/$(SWIFT_LAMBDA_LIBRARY)-arn.txt))
 	$(info "$(LAMBDA_LAYER_ARN)")
-	aws lambda create-function --function-name $(LAMBDA_FUNCTION_NAME) --runtime provided --handler $(LAMBDA_HANDLER) --role "$(IAM_ROLE_ARN)" --zip-file fileb://$(LAMBDA_BUILD_PATH)/$(LAMBDA_ZIP) --layers $(LAMBDA_LAYER_ARN) --profile $(AWS_PROFILE)
+	aws lambda create-function --function-name $(LAMBDA_FUNCTION_NAME) --runtime provided --handler $(LAMBDA_HANDLER) --role "$(IAM_ROLE_ARN)" --code "S3Bucket=$(AWS_BUCKET),S3Key=$(LAMBDA_S3_UPLOAD_PATH)/$(LAMBDA_ZIP)" --layers "$(LAMBDA_LAYER_ARN)" --profile $(AWS_PROFILE)
 
 update_lambda: package_lambda
 	aws lambda update-function-code --function-name $(LAMBDA_FUNCTION_NAME) --zip-file fileb://$(LAMBDA_BUILD_PATH)/$(LAMBDA_ZIP) --profile $(AWS_PROFILE)
 
+update_lambda_with_s3: package_lambda upload_build_to_s3
+	aws lambda update-function-code --function-name $(LAMBDA_FUNCTION_NAME) --s3-bucket $(AWS_BUCKET) --s3-key "$(LAMBDA_S3_UPLOAD_PATH)/$(LAMBDA_ZIP)" --profile $(AWS_PROFILE)
+
 invoke_lambda:
-	aws lambda invoke --function-name $(LAMBDA_FUNCTION_NAME) --profile $(AWS_PROFILE) --payload "fileb://$(SWIFT_PROJECT_PATH)/event.json" $(LAMBDA_BUILD_PATH)/outfile && echo "\nResult:" && cat $(LAMBDA_BUILD_PATH)/outfile && echo "\n"
+	aws lambda invoke --function-name $(LAMBDA_FUNCTION_NAME) --profile $(AWS_PROFILE) --payload "fileb://$(SWIFT_PROJECT_PATH)/event.json" $(TMP_BUILD_PATH)/outfile && echo "\nResult:" && cat $(TMP_BUILD_PATH)/outfile && echo "\n"
+
+create_s3_bucket:
+	aws s3 mb "s3://$(AWS_BUCKET)" --profile $(AWS_PROFILE)
 
 #quick commands - no clean
 quick_build_lambda: build_lambda create_build_directory
 	zip -r -j $(LAMBDA_BUILD_PATH)/$(LAMBDA_ZIP) $(SWIFT_PROJECT_PATH)/.build/$(SWIFT_CONFIGURATION)/$(SWIFT_EXECUTABLE)
 
 quick_deploy_lambda: quick_build_lambda create_build_directory
-	aws lambda update-function-code --function-name $(LAMBDA_FUNCTION_NAME) --zip-file fileb://$(LAMBDA_BUILD_PATH)/lambda.zip --profile $(AWS_PROFILE)
+	aws lambda update-function-code --function-name $(LAMBDA_FUNCTION_NAME) --zip-file fileb://$(LAMBDA_BUILD_PATH)/$(LAMBDA_ZIP) --profile $(AWS_PROFILE)

README.md

@@ -171,6 +171,8 @@ make invoke_lambda \
 | Key | Usage | Default |
 | --- | --- | --- |
 | AWS_PROFILE | An AWS AIM profile you create to authenticate to your account. | default |
+| IAM_ROLE_NAME | The execution role created that will be assumed by the Lambda. | lambda_sprinter_basic_execution |
+| AWS_BUCKET | The AWS S3 bucket where the layer and lambdas zip files get uploaded. | aws-lambda-swift-sprinter |
 | SWIFT_VERSION | Version of Swift used / Matches Dockerfile location too from `docker/` folder. | 5.1 |
 | LAYER_VERSION | Version of the Swift layer that will be created and uploaded for the Lambda to run on. | 5-1 |
 | SWIFT_EXECUTABLE | Name of the binary file. | HelloWorld |
@@ -215,6 +217,8 @@ There are many ways to achieve a lambda deployment (AWS Console, SAM, CloudForma
 - an AWS account for test purpose.
 - aws cli: Install the aws cli. Here the [instructions](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-welcome.html).
 
+- If you want to deploy your lambdas and layers using S3 you need to make sure the bucket in the Makefile already exists. If it doesn't you can create it using the command `make create_s3_bucket` which will use the value of the variable `AWS_BUCKET` as a name.
+
 - if your AWS account it doesn't have admin priviledges:
     - Review(*) the policy contained in the file **LambdaDeployerPolicyExample.json**
     - Attach the policy to your user. [Here](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_manage-attach-detach.html) is the official documentation.
@@ -224,17 +228,36 @@ It would be better to restrict the policy to the function and layer you want to
 It's suggested to test the following scripts before using in production.
 
 #### 1) Upload the Lambda layer
-Create a new lambda layer using the `swift-lambda-runtime.zip` file:
+Create a new lambda layer using the `swift-lambda-runtime.zip` file. This step is required once, however you are free to run it if you need to update your layer.
+
+##### Upload the Lambda using S3
 
 ```console
-make upload_lambda_layer
+make upload_lambda_layer_with_s3
 ```
 
-This step is required once, or if you need to update the layer.
+Datetime based versions are created and uploaded to S3 every time your version is created.
+
+##### Upload the Lambda directly
+
+```console
+make upload_lambda_layer
+```
 
 #### 2) Create the Lambda
 
-Create a new lambda, this might take a few minutes:
+You can create a new lambda which might take a few minutes using one of the options below:
+
+##### Create the Lambda using S3
+
+```console
+make create_lambda_with_s3
+```
+
+Datetime based versions are created and uploaded to S3 every time your version is created.
+
+##### Create the Lambda directly
+
 ```console
 make create_lambda
 ```
@@ -248,7 +271,7 @@ The lambda is created with the following parameters:
     - HelloWorld.helloWorld
 - role: `"$(IAM_ROLE_ARN)"`
     - a new role will be created with the name: `lambda_sprinter_basic_execution`
-- zip-file: $(LAMBDA_BUILD_PATH)/$(LAMBDA_ZIP) --> `./build/lambda.zip`
+- zip-file: $(LAMBDA_BUILD_PATH)/$(LAMBDA_ZIP) --> `./build/lambda/lambda.zip`
 - layers: `$(LAMBDA_LAYER_ARN)`
     - it will use the arn contained in the file generated by the `make upload_lambda_layer`
 
@@ -280,7 +303,15 @@ The lambda invocation may require some policy to access other AWS Resources. Che
 
 #### 5) Update the Lambda (optional)
 
-If it's required to update the lambda code, launch the following command:
+If needed, you will also be able to update your Lambda using one of the commands below:
+
+##### Update the Lambda using S3
+
+```console
+make update_lambda_with_s3
+```
+
+##### Update the Lambda directly
 
 ```console
 make update_lambda