Merge pull request #308 from NASA-IMPACT/dev

Preparation for Sentinel S2C platform

Author: sharkinsspatial

environment.sh.sample

@@ -1,9 +1,11 @@
 #!/bin/bash
 
 # Make sure that necessary executables are installed
-for b in jq aws
-do
-    command -v $b >/dev/null 2>&1 || { echo >&2 "I require $b but it's not installed.  Aborting."; exit 1; }
+for b in jq aws; do
+    command -v $b >/dev/null 2>&1 || {
+        echo >&2 "I require $b but it's not installed.  Aborting."
+        exit 1
+    }
 done
 
 # Set allexport mode, all variables defined in this block will get exported
@@ -19,15 +21,13 @@ HLS_LAADS_TOKEN="<your token>"
 HLS_OUTPUT_BUCKET=hls-global
 HLS_OUTPUT_BUCKET_HISTORIC=hls-gobal-historic
 
-
 # Role for copying to output bucket
 HLS_OUTPUT_BUCKET_ROLE_ARN=arn:aws:iam::611670965994:role/gcc-S3Test
 
 # Landsat SNS topic
 HLS_LANDSAT_SNS_TOPIC=arn:aws:sns:us-west-2:673253540267:public-c2-notify
 HLS_LANDASAT_HISTORIC_SNS_TOPIC=arn:aws:sns:us-west-2:018923174646:landsat-historic-LandsatHistoricTopic643F0596-1TIGFB893SX3B
 
-
 # Bucket for merged GIBS tile output.
 HLS_GIBS_OUTPUT_BUCKET=hls-browse-imagery
 HLS_LAADS_BUCKET_BOOTSTRAP=hls-development-laads-bucket
@@ -76,6 +76,9 @@ HLS_SSH_KEYNAME=hls-mount
 # Sentinel serverless downloader function role arn.
 HLS_DOWNLOADER_FUNCTION_ARN=something
 
+# Number of days after which objects in the Sentinel input buckets expire
+HLS_SENTINEL_INPUT_BUCKET_EXPIRATION_DAYS=60
+
 # GCC Specific environment settings.
 GCC=false
 HLS_GCC_ACCOUNT=account_id
@@ -89,13 +92,14 @@ HLS_GCC_BOUNDARY_ARN=boudary_policy_arn
 set +a
 
 # Set environment variables for all outputs set up in cloud formation
-stack_info=$(aws cloudformation describe-stacks --stack-name ${HLS_STACKNAME} --output json)
+stack_info=$(aws cloudformation describe-stacks --stack-name "${HLS_STACKNAME}" --output json)
 if [[ "$stack_info" =~ "OutputKey" ]]; then
     l=$(echo "$stack_info" | jq ".Stacks[].Outputs | length")
-    for ((i=0;i<$l;++i)); do
-        key=$(echo "$stack_info" | jq ".Stacks[].Outputs[$i].OutputKey" | sed -e 's/^"//'  -e 's/"$//')
+
+    for ((i = 0; i < l; ++i)); do
+        key=$(echo "$stack_info" | jq ".Stacks[].Outputs[$i].OutputKey" | sed -e 's/^"//' -e 's/"$//')
         keyupper=$(echo "$key" | awk '{print toupper($0)}')
-        val=$(echo "$stack_info" | jq ".Stacks[].Outputs[$i].OutputValue" | sed -e 's/^"//'  -e 's/"$//')
+        val=$(echo "$stack_info" | jq ".Stacks[].Outputs[$i].OutputValue" | sed -e 's/^"//' -e 's/"$//')
         export "HLSSTACK_$keyupper"="$val"
     done
 fi

setup.py

@@ -9,7 +9,9 @@
     "constructs>=10.0.0",
 ]
 
-install_requires: list[str] = []
+install_requires: list[str] = [
+    "setuptools>=64",
+]
 
 extras_require_test = [
     *aws_cdk_extras,

stack/stack.py

@@ -149,19 +149,45 @@ def __init__(self, scope: Construct, id: str, **kwargs) -> None:
             self, "landsat_output_bucket", OUTPUT_BUCKET
         )
 
+        sentinel_input_bucket_expiration_days = int(
+            os.environ["HLS_SENTINEL_INPUT_BUCKET_EXPIRATION_DAYS"]
+        )
+
         # Must be created as part of the stack due to trigger requirements
         self.sentinel_input_bucket = aws_s3.Bucket(
             self,
             "SentinelInputBucket",
             bucket_name=SENTINEL_INPUT_BUCKET,
             removal_policy=RemovalPolicy.DESTROY,
+            lifecycle_rules=[
+                # Setting expired_object_delete_marker cannot be done within a
+                # lifecycle rule that also specifies expiration, expiration_date, or
+                # tag_filters.
+                aws_s3.LifecycleRule(expired_object_delete_marker=True),
+                aws_s3.LifecycleRule(
+                    abort_incomplete_multipart_upload_after=Duration.days(1),
+                    expiration=Duration.days(sentinel_input_bucket_expiration_days),
+                    noncurrent_version_expiration=Duration.days(1),
+                ),
+            ],
         )
 
         self.sentinel_input_bucket_historic = aws_s3.Bucket(
             self,
             "SentinelInputBucketHistoric",
             bucket_name=SENTINEL_INPUT_BUCKET_HISTORIC,
             removal_policy=RemovalPolicy.DESTROY,
+            lifecycle_rules=[
+                # Setting expired_object_delete_marker cannot be done within a
+                # lifecycle rule that also specifies expiration, expiration_date, or
+                # tag_filters.
+                aws_s3.LifecycleRule(expired_object_delete_marker=True),
+                aws_s3.LifecycleRule(
+                    abort_incomplete_multipart_upload_after=Duration.days(1),
+                    expiration=Duration.days(sentinel_input_bucket_expiration_days),
+                    noncurrent_version_expiration=Duration.days(1),
+                ),
+            ],
         )
 
         self.landsat_input_bucket_historic = aws_s3.Bucket(
@@ -176,15 +202,25 @@ def __init__(self, scope: Construct, id: str, **kwargs) -> None:
             "LandsatIntermediateBucket",
             bucket_name=LANDSAT_INTERMEDIATE_OUTPUT_BUCKET,
             removal_policy=RemovalPolicy.DESTROY,
-            lifecycle_rules=[aws_s3.LifecycleRule(expiration=Duration.days(60))],
+            lifecycle_rules=[
+                aws_s3.LifecycleRule(
+                    abort_incomplete_multipart_upload_after=Duration.days(1),
+                    expiration=Duration.days(60),
+                )
+            ],
         )
 
         self.gibs_intermediate_output_bucket = aws_s3.Bucket(
             self,
             "GibsIntermediateBucket",
             bucket_name=GIBS_INTERMEDIATE_OUTPUT_BUCKET,
             removal_policy=RemovalPolicy.DESTROY,
-            lifecycle_rules=[aws_s3.LifecycleRule(expiration=Duration.days(60))],
+            lifecycle_rules=[
+                aws_s3.LifecycleRule(
+                    abort_incomplete_multipart_upload_after=Duration.days(1),
+                    expiration=Duration.days(60),
+                )
+            ],
         )
 
         self.efs = Efs(self, "Efs", network=self.network)

tox.ini

@@ -6,7 +6,7 @@ extras = test
 envdir = toxenv
 passenv = AWS_DEFAULT_REGION
 commands =
-  pip install -e ./layers/hls_lambda_layer/python
+  pip install --use-pep517 -e ./layers/hls_lambda_layer/python
   python -m pytest --cov=lambda_functions --ignore=node_modules --ignore=cdk.out
   flake8
 
@@ -17,8 +17,8 @@ passenv =
   HLS_*
   AWS_*
 commands =
-  nodeenv --node=20.17.0 --python-virtualenv
-  npm install -g aws-cdk@v2.155.0
+  nodeenv --node=lts --python-virtualenv
+  npm install -g aws-cdk@v2.*
   cdk --version
 
 [testenv:dev]