add batch stack

Author: geospatial-jeff

infra/app.py

@@ -7,22 +7,19 @@
 
 
 app = cdk.App()
-FiboaSdaStack(app, "FiboaSdaStackTesting",
+FiboaSdaStack(
+    app,
+    "FiboaSdaStackTesting",
     # If you don't specify 'env', this stack will be environment-agnostic.
     # Account/Region-dependent features and context lookups will not work,
     # but a single synthesized template can be deployed anywhere.
-
     # Uncomment the next line to specialize this stack for the AWS Account
     # and Region that are implied by the current CLI configuration.
-
-    #env=cdk.Environment(account=os.getenv('CDK_DEFAULT_ACCOUNT'), region=os.getenv('CDK_DEFAULT_REGION')),
-
+    # env=cdk.Environment(account=os.getenv('CDK_DEFAULT_ACCOUNT'), region=os.getenv('CDK_DEFAULT_REGION')),
     # Uncomment the next line if you know exactly what Account and Region you
     # want to deploy the stack to. */
-
-    #env=cdk.Environment(account='123456789012', region='us-east-1'),
-
+    # env=cdk.Environment(account='123456789012', region='us-east-1'),
     # For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html
-    )
+)
 
 app.synth()

infra/aws/fiboa_sda.py

@@ -1,34 +1,98 @@
 import os
 
 from aws_cdk import (
+    aws_ec2 as ec2,
+    aws_ecs as ecs,
+    aws_ecr_assets as ecr_assets,
     aws_lambda as _lambda,
-    aws_s3 as _s3,
+    aws_iam as iam,
+    aws_s3 as s3,
+    aws_batch as batch,
     aws_s3_notifications,
-    Stack, Duration
+    Stack,
+    Duration,
+    Size,
 )
 from constructs import Construct
 
-class FiboaSdaStack(Stack):
 
+class FiboaSdaStack(Stack):
     def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
         super().__init__(scope, construct_id, **kwargs)
 
-        # Create a new container image
-        ecr_image = _lambda.EcrImageCode.from_asset_image(
-            directory=os.path.join(os.getcwd(), "lambda-image")
+        # Create a VPC for the batch fargate cluster
+        vpc = ec2.Vpc(self, "VPC")
+
+        # Create AWS Batch Job Queue
+        self.batch_queue = batch.JobQueue(self, "JobQueue")
+        fargate_spot_environment = batch.FargateComputeEnvironment(
+            self,
+            "FargateSpotEnv",
+            vpc_subnets=ec2.SubnetSelection(
+                subnet_type=ec2.SubnetType.PRIVATE_WITH_NAT
+            ),
+            vpc=vpc,
+            spot=True,
+        )
+        self.batch_queue.add_compute_environment(fargate_spot_environment, 0)
+
+        # Task execution IAM role for Fargate
+        task_execution_role = iam.Role(
+            self,
+            "TaskExecutionRole",
+            assumed_by=iam.ServicePrincipal("ecs-tasks.amazonaws.com"),
+            managed_policies=[
+                iam.ManagedPolicy.from_aws_managed_policy_name(
+                    "service-role/AmazonECSTaskExecutionRolePolicy"
+                )
+            ],
+        )
+
+        # image_asset = ecr_assets.DockerImageAsset(
+        #     self, "MyImageAsset",
+        #     directory=os.path.join(os.getcwd(), "..")
+        # )
+
+        # Create Job Definition to submit job in batch job queue.
+        batch.EcsJobDefinition(
+            self,
+            "MyJobDef",
+            container=batch.EcsFargateContainerDefinition(
+                self,
+                "FargateCDKJobDef",
+                image=ecs.ContainerImage.from_asset(
+                    directory=os.path.join(os.getcwd(), "..")
+                ),
+                command=["ingest-one"],
+                memory=Size.gibibytes(16),
+                cpu=2,
+                execution_role=task_execution_role,
+            ),
         )
 
         # create lambda function
-        function = _lambda.Function(self, "fiboa-s3-listener",
-                                    runtime=_lambda.Runtime.FROM_IMAGE,
-                                    handler=_lambda.Handler.FROM_IMAGE,
-                                    architecture=_lambda.Architecture.ARM_64,
-                                    timeout=Duration.seconds(10), code=ecr_image)
+        # todo - create an IAM role with access to batch.
+        # todo - inject environment
+        function = _lambda.Function(
+            self,
+            "fiboa-s3-listener",
+            runtime=_lambda.Runtime.FROM_IMAGE,
+            handler=_lambda.Handler.FROM_IMAGE,
+            architecture=_lambda.Architecture.ARM_64,
+            timeout=Duration.seconds(10),
+            code=_lambda.EcrImageCode.from_asset_image(
+                directory=os.path.join(os.getcwd(), "lambda-image"),
+            ),
+        )
         # create s3 bucket
-        s3 = _s3.Bucket(self, "fiboa-sda-testing")
+        bucket = s3.Bucket(self, "fiboa-sda-testing")
 
         # create s3 notification for lambda function
         notification = aws_s3_notifications.LambdaDestination(function)
 
         # assign notification for the s3 event type (ex: OBJECT_CREATED)
-        s3.add_event_notification(_s3.EventType.OBJECT_CREATED, notification, _s3.NotificationKeyFilter(prefix="fiboa/*"))
+        bucket.add_event_notification(
+            s3.EventType.OBJECT_CREATED,
+            notification,
+            s3.NotificationKeyFilter(prefix="fiboa/*"),
+        )