Add local tests with Lambci and Localstack

Andrea-Scuderi · Andrea-Scuderi · commit 56a08b7e1c2e · 2019-12-14T13:55:26.000Z
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,6 @@
 .DS_Store
 /.build
+.tmp
 /Packages
 /*.xcodeproj
 lambda.zip
diff --git a/.travis.yml b/.travis.yml
@@ -23,4 +23,19 @@ script:
   - make package_lambda
   - make swift_test
   - make swift_test SWIFT_EXECUTABLE=HTTPSRequest SWIFT_PROJECT_PATH=Examples/HTTPSRequest LAMBDA_FUNCTION_NAME=HTTPSRequest LAMBDA_HANDLER=HTTPSRequest.getHttps
-  - make swift_test SWIFT_EXECUTABLE=S3Test SWIFT_PROJECT_PATH=Examples/S3Test LAMBDA_FUNCTION_NAME=S3Test LAMBDA_HANDLER=S3Test.getObject
+  - make swift_test SWIFT_EXECUTABLE=S3Test SWIFT_PROJECT_PATH=Examples/S3Test LAMBDA_FUNCTION_NAME=S3Test LAMBDA_HANDLER=S3Test.getObject
+  - make build_lambda_local
+  - make start_docker_compose_env
+  - make invoke_lambda_local
+  - make stop_docker_compose_env
+  - make test_lambda_local_output
+  - make build_lambda_local SWIFT_EXECUTABLE=HTTPSRequest SWIFT_PROJECT_PATH=Examples/HTTPSRequest LAMBDA_FUNCTION_NAME=HTTPSRequest LAMBDA_HANDLER=HTTPSRequest.getHttps
+  - make start_docker_compose_env SWIFT_EXECUTABLE=HTTPSRequest SWIFT_PROJECT_PATH=Examples/HTTPSRequest LAMBDA_FUNCTION_NAME=HTTPSRequest LAMBDA_HANDLER=HTTPSRequest.getHttps
+  - make invoke_lambda_local SWIFT_EXECUTABLE=HTTPSRequest SWIFT_PROJECT_PATH=Examples/HTTPSRequest LAMBDA_FUNCTION_NAME=HTTPSRequest LAMBDA_HANDLER=HTTPSRequest.getHttps
+  - make stop_docker_compose_env SWIFT_EXECUTABLE=HTTPSRequest SWIFT_PROJECT_PATH=Examples/HTTPSRequest LAMBDA_FUNCTION_NAME=HTTPSRequest LAMBDA_HANDLER=HTTPSRequest.getHttps
+  - make test_lambda_local_output SWIFT_EXECUTABLE=HTTPSRequest SWIFT_PROJECT_PATH=Examples/HTTPSRequest LAMBDA_FUNCTION_NAME=HTTPSRequest LAMBDA_HANDLER=HTTPSRequest.getHttps
+  - make build_lambda_local SWIFT_EXECUTABLE=S3Test SWIFT_PROJECT_PATH=Examples/S3Test LAMBDA_FUNCTION_NAME=S3Test LAMBDA_HANDLER=S3Test.getObject
+  - make start_docker_compose_env SWIFT_EXECUTABLE=S3Test SWIFT_PROJECT_PATH=Examples/S3Test LAMBDA_FUNCTION_NAME=S3Test LAMBDA_HANDLER=S3Test.getObject
+  - make invoke_lambda_local SWIFT_EXECUTABLE=S3Test SWIFT_PROJECT_PATH=Examples/S3Test LAMBDA_FUNCTION_NAME=S3Test LAMBDA_HANDLER=S3Test.getObject
+  - make stop_docker_compose_env SWIFT_EXECUTABLE=S3Test SWIFT_PROJECT_PATH=Examples/S3Test LAMBDA_FUNCTION_NAME=S3Test LAMBDA_HANDLER=S3Test.getObject
+  - make test_lambda_local_output SWIFT_EXECUTABLE=S3Test SWIFT_PROJECT_PATH=Examples/S3Test LAMBDA_FUNCTION_NAME=S3Test LAMBDA_HANDLER=S3Test.getObject
diff --git a/Examples/HTTPSRequest/Makefile b/Examples/HTTPSRequest/Makefile
@@ -0,0 +1,8 @@
+# Note: This must be launched from the root path
+SWIFT_PROJECT_PATH=Examples/HTTPSRequest
+
+start_docker_compose_env:
+	cd $(SWIFT_PROJECT_PATH); docker-compose up -d ; sleep 2
+
+stop_docker_compose_env:
+	cd $(SWIFT_PROJECT_PATH); docker-compose stop
diff --git a/Examples/HTTPSRequest/docker-compose.yml b/Examples/HTTPSRequest/docker-compose.yml
@@ -0,0 +1,16 @@
+version: '3'
+services:
+  lambci:
+    image: "lambci/lambda:provided"
+    volumes:
+        - ../../.build/local:/var/task:ro,delegated
+        - ../../bootstrap:/opt/bootstrap:ro,delegated
+        - ../../swift-shared-libs:/opt/swift-shared-libs:ro,delegated
+    environment:
+      - DOCKER_LAMBDA_STAY_OPEN=1
+      - AWS_LAMBDA_FUNCTION_TIMEOUT=3
+      - LAMB_CI_EXEC=1
+    command:
+      - HTTPSRequest.getHttps
+    ports:
+     - "9001:9001"
diff --git a/Examples/HTTPSRequest/event.json b/Examples/HTTPSRequest/event.json
@@ -1,3 +1,3 @@
 {
-    "url": "https://swift.org"
+    "url": "https://dog.ceo/api/breeds/list/all"
 }
diff --git a/Examples/HTTPSRequest/outfile.json b/Examples/HTTPSRequest/outfile.json
@@ -0,0 +1 @@
+{"url":"https:\/\/dog.ceo\/api\/breeds\/list\/all","content":"{\"message\":{\"affenpinscher\":[],\"african\":[],\"airedale\":[],\"akita\":[],\"appenzeller\":[],\"basenji\":[],\"beagle\":[],\"bluetick\":[],\"borzoi\":[],\"bouvier\":[],\"boxer\":[],\"brabancon\":[],\"briard\":[],\"buhund\":[\"norwegian\"],\"bulldog\":[\"boston\",\"english\",\"french\"],\"bullterrier\":[\"staffordshire\"],\"cairn\":[],\"cattledog\":[\"australian\"],\"chihuahua\":[],\"chow\":[],\"clumber\":[],\"cockapoo\":[],\"collie\":[\"border\"],\"coonhound\":[],\"corgi\":[\"cardigan\"],\"cotondetulear\":[],\"dachshund\":[],\"dalmatian\":[],\"dane\":[\"great\"],\"deerhound\":[\"scottish\"],\"dhole\":[],\"dingo\":[],\"doberman\":[],\"elkhound\":[\"norwegian\"],\"entlebucher\":[],\"eskimo\":[],\"frise\":[\"bichon\"],\"germanshepherd\":[],\"greyhound\":[\"italian\"],\"groenendael\":[],\"hound\":[\"afghan\",\"basset\",\"blood\",\"english\",\"ibizan\",\"walker\"],\"husky\":[],\"keeshond\":[],\"kelpie\":[],\"komondor\":[],\"kuvasz\":[],\"labrador\":[],\"leonberg\":[],\"lhasa\":[],\"malamute\":[],\"malinois\":[],\"maltese\":[],\"mastiff\":[\"bull\",\"english\",\"tibetan\"],\"mexicanhairless\":[],\"mix\":[],\"mountain\":[\"bernese\",\"swiss\"],\"newfoundland\":[],\"otterhound\":[],\"papillon\":[],\"pekinese\":[],\"pembroke\":[],\"pinscher\":[\"miniature\"],\"pitbull\":[],\"pointer\":[\"german\",\"germanlonghair\"],\"pomeranian\":[],\"poodle\":[\"miniature\",\"standard\",\"toy\"],\"pug\":[],\"puggle\":[],\"pyrenees\":[],\"redbone\":[],\"retriever\":[\"chesapeake\",\"curly\",\"flatcoated\",\"golden\"],\"ridgeback\":[\"rhodesian\"],\"rottweiler\":[],\"saluki\":[],\"samoyed\":[],\"schipperke\":[],\"schnauzer\":[\"giant\",\"miniature\"],\"setter\":[\"english\",\"gordon\",\"irish\"],\"sheepdog\":[\"english\",\"shetland\"],\"shiba\":[],\"shihtzu\":[],\"spaniel\":[\"blenheim\",\"brittany\",\"cocker\",\"irish\",\"japanese\",\"sussex\",\"welsh\"],\"springer\":[\"english\"],\"stbernard\":[],\"terrier\":[\"american\",\"australian\",\"bedlington\",\"border\",\"dandie\",\"fox\",\"irish\",\"kerryblue\",\"lakeland\",\"norfolk\",\"norwich\",\"patterdale\",\"russell\",\"scottish\",\"sealyham\",\"silky\",\"tibetan\",\"toy\",\"westhighland\",\"wheaten\",\"yorkshire\"],\"vizsla\":[],\"waterdog\":[\"spanish\"],\"weimaraner\":[],\"whippet\":[],\"wolfhound\":[\"irish\"]},\"status\":\"success\"}"}
diff --git a/Examples/HelloWorld/Makefile b/Examples/HelloWorld/Makefile
@@ -0,0 +1,8 @@
+# Note: This must be launched from the root path
+SWIFT_PROJECT_PATH=Examples/HelloWorld
+
+start_docker_compose_env:
+	cd $(SWIFT_PROJECT_PATH); docker-compose up -d ; sleep 2
+
+stop_docker_compose_env:
+	cd $(SWIFT_PROJECT_PATH); docker-compose stop
diff --git a/Examples/HelloWorld/docker-compose.yml b/Examples/HelloWorld/docker-compose.yml
@@ -0,0 +1,16 @@
+version: '3'
+services:
+  lambci:
+    image: "lambci/lambda:provided"
+    volumes:
+        - ../../.build/local:/var/task:ro,delegated
+        - ../../bootstrap:/opt/bootstrap:ro,delegated
+        - ../../swift-shared-libs:/opt/swift-shared-libs:ro,delegated
+    environment:
+      - DOCKER_LAMBDA_STAY_OPEN=1
+      - AWS_LAMBDA_FUNCTION_TIMEOUT=3
+      - LAMB_CI_EXEC=1
+    command:
+      - HelloWorld.helloWorld
+    ports:
+     - "9001:9001"
diff --git a/Examples/HelloWorld/outfile.json b/Examples/HelloWorld/outfile.json
@@ -0,0 +1 @@
+{"message":"Hello World! Hello Swift-Sprinter!"}
diff --git a/Examples/S3Test/Makefile b/Examples/S3Test/Makefile
@@ -0,0 +1,10 @@
+# Note: This must be launched from the root path
+SWIFT_PROJECT_PATH=Examples/S3Test
+
+start_docker_compose_env:
+	cd $(SWIFT_PROJECT_PATH); docker-compose up -d ; sleep 2
+	aws --endpoint http://localhost:4572 s3 mb s3://my-s3-bucket
+	aws --endpoint http://localhost:4572 s3 cp $(SWIFT_PROJECT_PATH)/hello.txt s3://my-s3-bucket/hello.txt
+
+stop_docker_compose_env:
+	cd $(SWIFT_PROJECT_PATH); docker-compose stop
diff --git a/Examples/S3Test/Sources/S3Test/main.swift b/Examples/S3Test/Sources/S3Test/main.swift
@@ -21,6 +21,7 @@ import Logging
 import S3
 import NIO
 import NIOFoundationCompat
+import AWSSDKSwiftCore
 
 struct Bucket: Codable {
     let name: String
@@ -31,10 +32,25 @@ struct Response: Codable {
     let value: String?
 }
 
-let s3 = S3(region: .useast1)
-
 let logger = Logger(label: "AWS.Lambda.S3Test")
 
+var s3: S3!
+
+if ProcessInfo.processInfo.environment["LAMB_CI_EXEC"] == "1" {
+    //Used for local test
+    s3 = S3(region: .useast1, endpoint: "http://localstack:4572")
+    logger.info("Endpoint-URI: http://localstack:4572")
+} else if let awsRegion = ProcessInfo.processInfo.environment["AWS_REGION"],
+    //The S3 Bucket must be in the same region of the Lambda
+    let region = Region(rawValue: awsRegion) {
+    s3 = S3(region: region)
+    logger.info("AWS_REGION: \(region)")
+} else {
+    //Default configuration
+    s3 = S3(region: .useast1)
+    logger.info("AWS_REGION: us-east-1")
+}
+
 /**
  How to use the `SyncCodableNIOLambda<Bucket, Response>` lambda handler with S3.
 
diff --git a/Examples/S3Test/docker-compose.yml b/Examples/S3Test/docker-compose.yml
@@ -0,0 +1,24 @@
+version: '3'
+services:
+  lambci:
+    image: "lambci/lambda:provided"
+    volumes:
+        - ../../.build/local:/var/task:ro,delegated
+        - ../../bootstrap:/opt/bootstrap:ro,delegated
+        - ../../swift-shared-libs:/opt/swift-shared-libs:ro,delegated
+    environment:
+      - DOCKER_LAMBDA_STAY_OPEN=1
+      - AWS_LAMBDA_FUNCTION_TIMEOUT=3
+      - LAMB_CI_EXEC=1
+    command:
+      - S3Test.getObject
+    ports:
+     - "9001:9001"
+  localstack:
+    image: localstack/localstack
+    ports:
+     - "4567-4584:4567-4584"
+    environment:
+     - SERVICES=s3
+    volumes:
+     - ".tmp:/tmp/localstack"
diff --git a/Examples/S3Test/outfile.json b/Examples/S3Test/outfile.json
@@ -0,0 +1 @@
+{"value":"Hello World!"}
diff --git a/Makefile b/Makefile
@@ -42,6 +42,8 @@ LAYER_ZIP=swift-lambda-runtime-$(LAYER_VERSION).zip
 ROOT_BUILD_PATH=./.build
 LAYER_BUILD_PATH=$(ROOT_BUILD_PATH)/layer
 LAMBDA_BUILD_PATH=$(ROOT_BUILD_PATH)/lambda
+LOCAL_LAMBDA_PATH=$(ROOT_BUILD_PATH)/local
+LOCALSTACK_TMP=$(ROOT_BUILD_PATH)/.tmp
 TMP_BUILD_PATH=$(ROOT_BUILD_PATH)/tmp
 DATETIME=$(shell date +'%y%m%d-%H%M%S')
 
@@ -207,3 +209,38 @@ quick_build_lambda: build_lambda create_build_directory
 
 quick_deploy_lambda: quick_build_lambda create_build_directory
 	aws lambda update-function-code --function-name $(LAMBDA_FUNCTION_NAME) --zip-file fileb://$(LAMBDA_BUILD_PATH)/$(LAMBDA_ZIP) --profile $(AWS_PROFILE)
+
+build_lambda_local: package_layer build_lambda
+	if [ ! -d "$(LOCAL_LAMBDA_PATH)" ]; then mkdir -p $(LOCAL_LAMBDA_PATH); fi
+	cp $(SWIFT_PROJECT_PATH)/.build/$(SWIFT_CONFIGURATION)/$(SWIFT_EXECUTABLE) $(LOCAL_LAMBDA_PATH)/.
+
+invoke_lambda_local_once:
+	$(eval LOCAL_LAMBDA_EVENT := '$(shell cat $(SWIFT_PROJECT_PATH)/event.json)')
+	docker run --rm \
+	-v "$(PWD)/.build/local":/var/task:ro,delegated \
+	-v "$(PWD)/bootstrap":/opt/bootstrap:ro,delegated \
+	-v "$(PWD)/swift-shared-libs":/opt/swift-shared-libs:ro,delegated \
+	lambci/lambda:provided $(LAMBDA_HANDLER) $(LOCAL_LAMBDA_EVENT)
+
+start_lambda_local_env:
+	docker run --rm \
+	-e DOCKER_LAMBDA_STAY_OPEN=1 \
+	-p 9001:9001 \
+	-v "$(PWD)/.build/local":/var/task:ro,delegated \
+	-v "$(PWD)/bootstrap":/opt/bootstrap:ro,delegated \
+	-v "$(PWD)/swift-shared-libs":/opt/swift-shared-libs:ro,delegated \
+  	lambci/lambda:provided \
+	$(LAMBDA_HANDLER)
+
+invoke_lambda_local:
+	aws lambda invoke --endpoint http://localhost:9001 --no-sign-request --function-name $(LAMBDA_FUNCTION_NAME) --payload "fileb://$(SWIFT_PROJECT_PATH)/event.json" $(TMP_BUILD_PATH)/outfile && echo "\nResult:" && cat $(TMP_BUILD_PATH)/outfile && echo "\n"
+
+start_docker_compose_env:
+	if [ ! -d "$(LOCALSTACK_TMP)" ]; then mkdir -p $(LOCALSTACK_TMP); fi
+	make -f $(SWIFT_PROJECT_PATH)/Makefile start_docker_compose_env
+
+stop_docker_compose_env:
+	make -f $(SWIFT_PROJECT_PATH)/Makefile stop_docker_compose_env
+
+test_lambda_local_output:
+	cmp $(TMP_BUILD_PATH)/outfile $(SWIFT_PROJECT_PATH)/outfile.json

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,3 @@`
`1`	`1`	`{`
`2`		`- "url": "https://swift.org"`
	`2`	`+ "url": "https://dog.ceo/api/breeds/list/all"`
`3`	`3`	`}`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+{"url":"https:\/\/dog.ceo\/api\/breeds\/list\/all","content":"{\"message\":{\"affenpinscher\":[],\"african\":[],\"airedale\":[],\"akita\":[],\"appenzeller\":[],\"basenji\":[],\"beagle\":[],\"bluetick\":[],\"borzoi\":[],\"bouvier\":[],\"boxer\":[],\"brabancon\":[],\"briard\":[],\"buhund\":[\"norwegian\"],\"bulldog\":[\"boston\",\"english\",\"french\"],\"bullterrier\":[\"staffordshire\"],\"cairn\":[],\"cattledog\":[\"australian\"],\"chihuahua\":[],\"chow\":[],\"clumber\":[],\"cockapoo\":[],\"collie\":[\"border\"],\"coonhound\":[],\"corgi\":[\"cardigan\"],\"cotondetulear\":[],\"dachshund\":[],\"dalmatian\":[],\"dane\":[\"great\"],\"deerhound\":[\"scottish\"],\"dhole\":[],\"dingo\":[],\"doberman\":[],\"elkhound\":[\"norwegian\"],\"entlebucher\":[],\"eskimo\":[],\"frise\":[\"bichon\"],\"germanshepherd\":[],\"greyhound\":[\"italian\"],\"groenendael\":[],\"hound\":[\"afghan\",\"basset\",\"blood\",\"english\",\"ibizan\",\"walker\"],\"husky\":[],\"keeshond\":[],\"kelpie\":[],\"komondor\":[],\"kuvasz\":[],\"labrador\":[],\"leonberg\":[],\"lhasa\":[],\"malamute\":[],\"malinois\":[],\"maltese\":[],\"mastiff\":[\"bull\",\"english\",\"tibetan\"],\"mexicanhairless\":[],\"mix\":[],\"mountain\":[\"bernese\",\"swiss\"],\"newfoundland\":[],\"otterhound\":[],\"papillon\":[],\"pekinese\":[],\"pembroke\":[],\"pinscher\":[\"miniature\"],\"pitbull\":[],\"pointer\":[\"german\",\"germanlonghair\"],\"pomeranian\":[],\"poodle\":[\"miniature\",\"standard\",\"toy\"],\"pug\":[],\"puggle\":[],\"pyrenees\":[],\"redbone\":[],\"retriever\":[\"chesapeake\",\"curly\",\"flatcoated\",\"golden\"],\"ridgeback\":[\"rhodesian\"],\"rottweiler\":[],\"saluki\":[],\"samoyed\":[],\"schipperke\":[],\"schnauzer\":[\"giant\",\"miniature\"],\"setter\":[\"english\",\"gordon\",\"irish\"],\"sheepdog\":[\"english\",\"shetland\"],\"shiba\":[],\"shihtzu\":[],\"spaniel\":[\"blenheim\",\"brittany\",\"cocker\",\"irish\",\"japanese\",\"sussex\",\"welsh\"],\"springer\":[\"english\"],\"stbernard\":[],\"terrier\":[\"american\",\"australian\",\"bedlington\",\"border\",\"dandie\",\"fox\",\"irish\",\"kerryblue\",\"lakeland\",\"norfolk\",\"norwich\",\"patterdale\",\"russell\",\"scottish\",\"sealyham\",\"silky\",\"tibetan\",\"toy\",\"westhighland\",\"wheaten\",\"yorkshire\"],\"vizsla\":[],\"waterdog\":[\"spanish\"],\"weimaraner\":[],\"whippet\":[],\"wolfhound\":[\"irish\"]},\"status\":\"success\"}"}
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+{"message":"Hello World! Hello Swift-Sprinter!"}`