[integ-tests] Improve ParallelCluster cfn custom resource integration tests

1. Extract cluster configuration file from `cluster_custom_resource` to individual pcluster.config.yaml files. Therefore, the tests are more similar/extensible to other integration tests and the cluster configuration can be easily modified for each test.
2. Let test_cluster_create and test_cluster_update use 50 queues. This will test most potential limits (e.g. Eventbridge size limit)
3. Simplify test_cluster_create_invalid and test_cluster_update_invalid. This will save the cost and time of integration tests
4. Reorder some fixtures, add a wait_for_rollback, add resource cleanup to fix sporadic failures and resource leftovers

Signed-off-by: Hanwen <[email protected]>

Author: hanwen-cluster

tests/integration-tests/cfn_stacks_factory.py

@@ -236,7 +236,16 @@ def delete_stack(self, name, region):
         wait_fixed=5000,
         retry_on_exception=lambda exception: isinstance(exception, ClientError),
     )
-    def update_stack(self, name, region, parameters, stack_is_under_test=False, tags=None, wait_for_rollback=False):
+    def update_stack(
+        self,
+        name,
+        region,
+        parameters,
+        stack_is_under_test=False,
+        tags=None,
+        template_body=None,
+        wait_for_rollback=False,
+    ):
         """Update a created cfn stack."""
         with aws_credential_provider(region, self.__credentials):
             internal_id = self.__get_stack_internal_id(name, region)
@@ -245,12 +254,11 @@ def update_stack(self, name, region, parameters, stack_is_under_test=False, tags
                 try:
                     stack = self.__created_stacks[internal_id]
                     cfn_client = boto3.client("cloudformation", region_name=stack.region)
+                    template_args = {"TemplateBody": template_body} if template_body else {"UsePreviousTemplate": True}
                     if tags is not None:
-                        cfn_client.update_stack(
-                            StackName=stack.name, UsePreviousTemplate=True, Parameters=parameters, Tags=tags
-                        )
+                        cfn_client.update_stack(StackName=stack.name, Parameters=parameters, Tags=tags, **template_args)
                     else:
-                        cfn_client.update_stack(StackName=stack.name, UsePreviousTemplate=True, Parameters=parameters)
+                        cfn_client.update_stack(StackName=stack.name, Parameters=parameters, **template_args)
 
                     if wait_for_rollback:
                         final_status = self.__wait_for_stack_update_rollback(stack.cfn_stack_id, cfn_client)
@@ -322,11 +330,11 @@ def __wait_for_stack_update(self, name, cfn_client):
         return self.__get_stack_status(name, cfn_client)
 
     @retry(
-        stop_max_attempt_number=15,
+        stop_max_attempt_number=30,
         retry_on_result=lambda result: result == "UPDATE_ROLLBACK_IN_PROGRESS"
         or result == "UPDATE_IN_PROGRESS"
         or result == "UPDATE_FAILED",
-        wait_fixed=5000,
+        wait_fixed=10000,
         retry_on_exception=lambda exception: isinstance(exception, ClientError) and "Rate exceeded" in str(exception),
     )
     def __wait_for_stack_update_rollback(self, name, cfn_client):

tests/integration-tests/configs/develop.yaml

@@ -105,31 +105,39 @@ test-suites:
       dimensions:
         - oss: ["alinux2"]
           regions: ["us-east-2"]
+          instances: {{ common.INSTANCES_DEFAULT_X86 }}
     test_cluster_custom_resource.py::test_cluster_create_invalid:
       dimensions:
         - oss: ["alinux2"]
           regions: ["us-east-2"]
+          instances: {{ common.INSTANCES_DEFAULT_X86 }}
     test_cluster_custom_resource.py::test_cluster_update:
       dimensions:
         - oss: ["alinux2"]
           regions: ["us-east-2"]
+          instances: {{ common.INSTANCES_DEFAULT_X86 }}
     test_cluster_custom_resource.py::test_cluster_update_invalid:
       dimensions:
         - oss: ["alinux2"]
           regions: ["us-east-2"]
+          instances: {{ common.INSTANCES_DEFAULT_X86 }}
     test_cluster_custom_resource.py::test_cluster_update_tag_propagation:
       dimensions:
         - oss: [ "alinux2" ]
           regions: ["us-east-2"]
+          instances: {{ common.INSTANCES_DEFAULT_X86 }}
     test_cluster_custom_resource.py::test_cluster_delete_out_of_band:
       dimensions:
         - oss: ["alinux2"]
           regions: ["us-east-2"]
+          instances: {{ common.INSTANCES_DEFAULT_X86 }}
     test_cluster_custom_resource.py::test_cluster_delete_retain:
       dimensions:
         - oss: ["alinux2"]
           regions: ["us-east-2"]
+          instances: {{ common.INSTANCES_DEFAULT_X86 }}
     test_cluster_custom_resource.py::test_cluster_create_with_custom_policies:
       dimensions:
         - oss: ["alinux2"]
           regions: ["us-east-2"]
+          instances: {{ common.INSTANCES_DEFAULT_X86 }}

tests/integration-tests/resources/cluster_custom_resource.yaml

@@ -5,89 +5,16 @@ Parameters:
   ClusterName:
     Description: Name of cluster. Note this must be different than the stack name.
     Type: String
-  HeadNodeSubnet:
-    Description: Subnet for the HeadNode
-    Type: String
-  ComputeNodeSubnet:
-    Description: Subnet for the ComputeNode
-    Type: String
-  ComputeInstanceMax:
-    Description: Maximum number of compute instances
-    Type: Number
-    Default: 16
   ServiceToken:
     Description: ARN of Lambda Function backing the Cluster Resource
     Type: String
-  Os:
-    Description: Operating system for nodes
-    Type: String
-    Default: 'alinux2'
-  OnNodeConfigured:
-    Description: Script to run on HeadNode configured
-    Type: String
-    Default: ''
-  CustomBucketAccess:
-    Description: Name of a bucket to provide access to on the HeadNode
-    Type: String
-    Default: ''
-  DeletionPolicy:
-    Type: String
-    Default: Delete
-    AllowedValues:
-      - Delete
-      - Retain
-    Description: Enter Retain or Delete to define the operation when the stack is deleted. Default is to Delete.
-
-Conditions:
-  OnNodeConfiguredCondition: !Not [!Equals [!Ref OnNodeConfigured, '']]
-  CustomBucketCondition: !Not [!Equals [!Ref CustomBucketAccess, '']]
 
 Resources:
   PclusterCluster:
     Type: Custom::PclusterCluster
     Properties:
       ServiceToken: !Ref ServiceToken
-      DeletionPolicy: !Ref DeletionPolicy
       ClusterName: !Ref ClusterName
-      ClusterConfiguration:
-        Imds:
-          ImdsSupport: v2.0
-        Tags:
-          - Key: inside_configuration_key
-            Value: overridden
-        DevSettings:
-          AmiSearchFilters:
-            Owner: self
-        Image:
-          Os: !Ref Os
-        HeadNode:
-          InstanceType: t2.small
-          Networking:
-            SubnetId: !Ref HeadNodeSubnet
-          CustomActions: !If
-            - OnNodeConfiguredCondition
-            -
-              OnNodeConfigured:
-                Script: !Ref OnNodeConfigured
-            - !Ref AWS::NoValue
-          Iam: !If
-            - CustomBucketCondition
-            -
-              S3Access:
-                - BucketName: !Ref CustomBucketAccess
-                  EnableWriteAccess: false
-            - !Ref AWS::NoValue
-        Scheduling:
-          Scheduler: slurm
-          SlurmQueues:
-          - Name: queue0
-            ComputeResources:
-            - Name: queue0-cr0
-              InstanceType: t2.micro
-              MaxCount: !Ref ComputeInstanceMax
-            Networking:
-              SubnetIds:
-              - !Ref ComputeNodeSubnet
 
 Outputs:
   HeadNodeIp:

tests/integration-tests/tests/custom_resource/conftest.py

@@ -17,6 +17,7 @@
 import cfn_tools
 import pkg_resources
 import pytest
+import yaml
 from cfn_stacks_factory import CfnStack
 from troposphere import Output, Ref
 from troposphere.iam import ManagedPolicy
@@ -74,10 +75,10 @@ def cluster_custom_resource_provider_generator(cfn_stacks_factory, region, stack
 
 @pytest.fixture(scope="class", name="cluster_custom_resource_provider")
 def cluster_custom_resource_provider_fixture(
-    cfn_stacks_factory,
     request,
     region,
     resource_bucket,
+    cfn_stacks_factory,
     cluster_custom_resource_service_token,
     cluster_custom_resource_provider_template,
 ):
@@ -115,35 +116,43 @@ def cluster_1_click_fixture(cfn_stacks_factory, request, region, key_name, clust
     return stack
 
 
+def get_custom_resource_template(cluster_config_path, cluster_custom_resource_template, deletion_policy="Delete"):
+    with open(cluster_custom_resource_template, "r", encoding="utf-8") as f:
+        template = TemplateGenerator(cfn_tools.load_yaml(f.read()))
+    with open(cluster_config_path, encoding="utf-8") as cluster_config:
+        template.resources["PclusterCluster"].properties["ClusterConfiguration"] = yaml.safe_load(cluster_config.read())
+    template.resources["PclusterCluster"].properties["DeletionPolicy"] = deletion_policy
+    return template
+
+
 @pytest.fixture(scope="class", name="cluster_custom_resource_factory")
 def cluster_custom_resource_factory_fixture(
-    cfn_stacks_factory,
     request,
     region,
-    os,
-    cluster_custom_resource_template,
     cluster_custom_resource_provider,
     vpc_stack,
+    cfn_stacks_factory,
+    cluster_custom_resource_template,
 ):
-    def _produce_cluster_custom_resource_stack(parameters=None):
-        cluster_name = generate_stack_name("integ-test-custom-resource-c", request.config.getoption("stackname_suffix"))
+    created_stacks = []
 
-        parameters = {
-            "ClusterName": cluster_name,
-            "HeadNodeSubnet": vpc_stack.get_public_subnet(),
-            "ComputeNodeSubnet": vpc_stack.get_private_subnet(),
-            "ServiceToken": cluster_custom_resource_provider,
-            "Os": os,
-            **(parameters or {}),
-        }
+    def _produce_cluster_custom_resource_stack(
+        cluster_config_path, cluster_name=None, deletion_policy="Delete", service_token=None
+    ):
+        cluster_name = cluster_name or generate_stack_name(
+            "integ-test-custom-resource-c", request.config.getoption("stackname_suffix")
+        )
 
-        with open(cluster_custom_resource_template, encoding="utf-8") as cfn_file:
-            template_data = cfn_file.read()
+        parameters = {"ClusterName": cluster_name, "ServiceToken": service_token or cluster_custom_resource_provider}
+
+        template = get_custom_resource_template(
+            cluster_config_path, cluster_custom_resource_template, deletion_policy=deletion_policy
+        )
 
         stack = CfnStack(
             name=generate_stack_name("integ-tests-custom-resource", request.config.getoption("stackname_suffix")),
             region=region,
-            template=template_data,
+            template=template.to_yaml(),
             parameters=[{"ParameterKey": k, "ParameterValue": v} for k, v in parameters.items()],
             capabilities=["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM", "CAPABILITY_AUTO_EXPAND"],
             tags=[
@@ -154,9 +163,13 @@ def _produce_cluster_custom_resource_stack(parameters=None):
 
         cfn_stacks_factory.create_stack(stack, True)
         stack.factory = cfn_stacks_factory
+        created_stacks.append(stack)
         return stack
 
     yield _produce_cluster_custom_resource_stack
+    if not request.config.getoption("no_delete"):
+        for stack in created_stacks:
+            stack.factory.delete_stack(stack.name, region)
 
 
 @pytest.fixture(scope="class", name="resource_bucket_cluster_template")