Add integration test for cluster_resource_bucket

* Add creation/deletion test for cluster_resource_bucket parameter
* Modified update test to use cluster_resource_bucket for test_update_hit

Signed-off-by: Rex <[email protected]>

Author: Rex

tests/integration-tests/configs/develop.yaml

@@ -543,4 +543,11 @@ test-suites:
         - regions: ["us-east-1"]
           instances: ["p4d.24xlarge"]
           oss: ["alinux", "ubuntu1804", "centos7"]
-          schedulers: ["slurm"]
+          schedulers: ["slurm"]
+  resource_bucket:
+    test_resource_bucket.py::test_resource_bucket:
+      dimensions:
+        - regions: ["ap-southeast-1"]
+          instances: {{ common.INSTANCES_DEFAULT_X86 }}
+          oss: ["alinux2"]
+          schedulers: ["slurm", "awsbatch"]

tests/integration-tests/tests/resource_bucket/test_resource_bucket.py

@@ -0,0 +1,57 @@
+import logging
+
+import boto3
+import pytest
+from assertpy import assert_that
+
+
+@pytest.mark.regions(["us-east-2"])
+@pytest.mark.schedulers(["slurm", "awsbatch"])
+@pytest.mark.oss(["alinux"])
+@pytest.mark.usefixtures("os", "instance")
+def test_resource_bucket(region, scheduler, pcluster_config_reader, clusters_factory, s3_bucket_factory, test_datadir):
+    # Bucket used to host cluster artifacts must have versioning enabled
+    logging.info("Testing cluster creation/deletion behavior when specifying cluster_resource_bucket")
+    bucket_name = s3_bucket_factory()
+    # Upload a file to bucket, we will check to make sure this file is not removed when deleting cluster artifacts
+    boto3.resource("s3").Bucket(bucket_name).upload_file(str(test_datadir / "s3_test_file"), "s3_test_file")
+
+    cluster_config = pcluster_config_reader(
+        config_file="pcluster.config_{0}.ini".format(scheduler), resource_bucket=bucket_name
+    )
+    cluster = clusters_factory(cluster_config)
+    assert_that(cluster.cfn_outputs.get("ResourcesS3Bucket")).is_equal_to(bucket_name)
+    artifact_directory = cluster.cfn_outputs.get("ArtifactS3RootDirectory")
+    assert_that(artifact_directory).is_not_none()
+    # Update cluster with a new resource bucket
+    # We need to make sure the bucket name in cfn NEVER gets updated
+    update_bucket_name = s3_bucket_factory()
+    updated_config_file = pcluster_config_reader(
+        config_file="pcluster.config_{0}.ini".format(scheduler), resource_bucket=update_bucket_name
+    )
+    cluster.config_file = str(updated_config_file)
+    cluster.update()
+    assert_that(cluster.cfn_outputs.get("ResourcesS3Bucket")).is_equal_to(bucket_name)
+    assert_that(cluster.cfn_outputs.get("ArtifactS3RootDirectory")).is_equal_to(artifact_directory)
+
+    cluster.delete()
+    _check_delete_behavior(region, bucket_name, artifact_directory)
+
+
+def _check_delete_behavior(region, bucket_name, artifact_directory):
+    s3_client = boto3.client("s3", region_name=region)
+    response = s3_client.list_objects_v2(Bucket=bucket_name, Delimiter="/", Prefix=artifact_directory)
+    if response.get("Contents") or response.get("CommonPrefixes"):
+        logging.error(
+            "Objects under %s/%s not cleaned up properly!\nContents: %s\nCommonPrefixes: %s",
+            bucket_name,
+            artifact_directory,
+            response.get("Contents"),
+            response.get("CommonPrefixes"),
+        )
+        raise Exception
+    try:
+        s3_client.head_object(Bucket=bucket_name, Key="s3_test_file")
+    except Exception as e:
+        logging.error("Unable to verify pre-existing files in bucket are preserved, with error: %s", e)
+        raise

tests/integration-tests/tests/resource_bucket/test_resource_bucket/test_resource_bucket/pcluster.config_awsbatch.ini

@@ -0,0 +1,21 @@
+[global]
+cluster_template = default
+
+[aws]
+aws_region_name = {{ region }}
+
+[vpc parallelcluster-vpc]
+vpc_id = {{ vpc_id }}
+master_subnet_id = {{ public_subnet_id }}
+compute_subnet_id = {{ private_subnet_id }}
+use_public_ips = false
+
+[cluster default]
+base_os = {{ os }}
+key_name = {{ key_name }}
+vpc_settings = parallelcluster-vpc
+scheduler = {{ scheduler }}
+master_instance_type = {{ instance }}
+compute_instance_type = {{ instance }}
+min_vcpus=1
+cluster_resource_bucket = {{ resource_bucket }}

tests/integration-tests/tests/resource_bucket/test_resource_bucket/test_resource_bucket/pcluster.config_slurm.ini

@@ -0,0 +1,28 @@
+[global]
+cluster_template = default
+
+[aws]
+aws_region_name = {{ region }}
+
+[vpc parallelcluster-vpc]
+vpc_id = {{ vpc_id }}
+master_subnet_id = {{ public_subnet_id }}
+compute_subnet_id = {{ private_subnet_id }}
+use_public_ips = false
+
+[cluster default]
+base_os = {{ os }}
+key_name = {{ key_name }}
+vpc_settings = parallelcluster-vpc
+scheduler = {{ scheduler }}
+master_instance_type = {{ instance }}
+cluster_resource_bucket = {{ resource_bucket }}
+queue_settings = queue1
+
+[queue queue1]
+compute_resource_settings = queue1_i1
+compute_type = ondemand
+
+[compute_resource queue1_i1]
+instance_type = {{ instance }}
+min_count = 1

tests/integration-tests/tests/resource_bucket/test_resource_bucket/test_resource_bucket/s3_test_file

@@ -0,0 +1 @@
+# Sample test file to make sure existing files in S3 bucket are not removed on when cleaning up cluster resources

tests/integration-tests/tests/update/test_update.py

@@ -42,7 +42,7 @@ def test_update_sit(
     bucket.upload_file(str(test_datadir / "postinstall.sh"), "scripts/postinstall.sh")
 
     # Create cluster with initial configuration
-    init_config_file = pcluster_config_reader()
+    init_config_file = pcluster_config_reader(resource_bucket=bucket_name)
     cluster = clusters_factory(init_config_file)
 
     # Update cluster with the same configuration, command should not result any error even if not using force update
@@ -148,7 +148,7 @@ def test_update_hit(region, scheduler, pcluster_config_reader, clusters_factory,
     bucket.upload_file(str(test_datadir / "postinstall.sh"), "scripts/postinstall.sh")
 
     # Create cluster with initial configuration
-    init_config_file = pcluster_config_reader()
+    init_config_file = pcluster_config_reader(resource_bucket=bucket_name)
     cluster = clusters_factory(init_config_file)
 
     # Update cluster with the same configuration, command should not result any error even if not using force update
@@ -204,7 +204,9 @@ def test_update_hit(region, scheduler, pcluster_config_reader, clusters_factory,
     job_id = slurm_commands.assert_job_submitted(result.stdout)
 
     # Update cluster with new configuration
-    updated_config_file = pcluster_config_reader(config_file="pcluster.config.update.ini", bucket=bucket_name)
+    updated_config_file = pcluster_config_reader(
+        config_file="pcluster.config.update.ini", bucket=bucket_name, resource_bucket=bucket_name
+    )
     cluster.config_file = str(updated_config_file)
     cluster.update()
 

tests/integration-tests/tests/update/test_update/test_update_hit/pcluster.config.ini

@@ -30,6 +30,7 @@ raid_settings = custom
 fsx_settings = custom
 cw_log_settings = custom
 queue_settings = queue1,queue2
+cluster_resource_bucket = {{ resource_bucket }}
 
 [queue queue1]
 compute_resource_settings = queue1_i1,queue1_i2

tests/integration-tests/tests/update/test_update/test_update_hit/pcluster.config.update.ini

@@ -29,6 +29,7 @@ raid_settings = custom
 fsx_settings = custom
 cw_log_settings = custom
 queue_settings = queue1,queue2,queue3
+cluster_resource_bucket = {{ resource_bucket }}
 
 [queue queue1]
 compute_resource_settings = queue1_i1,queue1_i2,queue1_i3

tests/integration-tests/utils.py

@@ -195,6 +195,8 @@ def create_s3_bucket(bucket_name, region):
         s3_client.create_bucket(Bucket=bucket_name, CreateBucketConfiguration={"LocationConstraint": region})
     else:
         s3_client.create_bucket(Bucket=bucket_name)
+    # Enable versioning on bucket
+    s3_client.put_bucket_versioning(Bucket=bucket_name, VersioningConfiguration={"Status": "Enabled"})
 
 
 @retry(wait_exponential_multiplier=500, wait_exponential_max=5000, stop_max_attempt_number=3)
@@ -208,6 +210,7 @@ def delete_s3_bucket(bucket_name, region):
     try:
         bucket = boto3.resource("s3", region_name=region).Bucket(bucket_name)
         bucket.objects.all().delete()
+        bucket.object_versions.all().delete()
         bucket.delete()
     except boto3.client("s3").exceptions.NoSuchBucket:
         pass