[CLI] Add CreateFleet instance info retrieval timeout configurable via DevSettings (#7425)

* Add DevSettings/Timeouts ComputeInstanceInfoTimeout to cluster config schema

Expose the CreateFleet post-launch DescribeInstances retry timeout (consumed by
the node package to tolerate EC2 API eventual consistency) as a dev setting under
DevSettings/Timeouts/ComputeInstanceInfoTimeout, defaulting to 90s.

Author: hehe7318

CHANGELOG.md

@@ -24,6 +24,7 @@ CHANGELOG
   which used to cause cluster update failure on clusters with login nodes.
 - Fail `pcluster build-image` early when the downloaded cookbook version does not match the ParallelCluster CLI version.
 - Fix login nodes not mounting `/opt/parallelcluster/shared` when EFS is used as the internal shared storage type.
+- Fix an issue where compute nodes are incorrectly replaced when launching a large number of nodes due to eventual consistency.
 
 **DEPRECATIONS**
 - Amazon Linux 2 is no longer supported.

cli/src/pcluster/config/cluster_config.py

@@ -36,6 +36,7 @@
 )
 from pcluster.constants import (
     CIDR_ALL_IPS,
+    COMPUTE_INSTANCE_INFO_TIMEOUT,
     CW_ALARMS_ENABLED_DEFAULT,
     CW_DASHBOARD_ENABLED_DEFAULT,
     CW_LOGS_ENABLED_DEFAULT,
@@ -1189,14 +1190,22 @@ def __init__(self, tags: List[Tag] = None, owner: str = None, name_prefix: str =
 class Timeouts(Resource):
     """Represent the configuration for node boostrap timeout."""
 
-    def __init__(self, head_node_bootstrap_timeout: int = None, compute_node_bootstrap_timeout: int = None):
+    def __init__(
+        self,
+        head_node_bootstrap_timeout: int = None,
+        compute_node_bootstrap_timeout: int = None,
+        compute_instance_info_timeout: int = None,
+    ):
         super().__init__()
         self.head_node_bootstrap_timeout = Resource.init_param(
             head_node_bootstrap_timeout, default=NODE_BOOTSTRAP_TIMEOUT
         )
         self.compute_node_bootstrap_timeout = Resource.init_param(
             compute_node_bootstrap_timeout, default=NODE_BOOTSTRAP_TIMEOUT
         )
+        self.compute_instance_info_timeout = Resource.init_param(
+            compute_instance_info_timeout, default=COMPUTE_INSTANCE_INFO_TIMEOUT
+        )
 
 
 class CapacityReservationTarget(Resource):

cli/src/pcluster/constants.py

@@ -236,6 +236,10 @@
 
 NODE_BOOTSTRAP_TIMEOUT = 2100
 
+# Default time budget (seconds) for retrieving EC2 instance info after a CreateFleet launch,
+# to tolerate EC2 API eventual consistency.
+COMPUTE_INSTANCE_INFO_TIMEOUT = 90
+
 # DirectoryService
 DIRECTORY_SERVICE_RESERVED_SETTINGS = {"id_provider": "ldap"}
 

cli/src/pcluster/schemas/cluster_schema.py

@@ -1103,6 +1103,9 @@ class TimeoutsSchema(BaseSchema):
     compute_node_bootstrap_timeout = fields.Int(
         validate=validate.Range(min=1), metadata={"update_policy": UpdatePolicy.SUPPORTED}
     )
+    compute_instance_info_timeout = fields.Int(
+        validate=validate.Range(min=1), metadata={"update_policy": UpdatePolicy.SUPPORTED}
+    )
 
     @post_load()
     def make_resource(self, data, **kwargs):

cli/tests/pcluster/example_configs/slurm.full.yaml

@@ -329,6 +329,7 @@ DevSettings:
   Timeouts:
     HeadNodeBootstrapTimeout: 1201  # Default 1800 (seconds)
     ComputeNodeBootstrapTimeout: 1001  # Default 1800 (seconds)
+    ComputeInstanceInfoTimeout: 95  # Default 90 (seconds)
   ComputeStartupTimeMetricEnabled: false
 DeploymentSettings:
   DisableSudoAccessForDefaultUser: True