Upgrade Karpenter & K8s version; delete cloud formation templates properly

Karpenter runs started failing, as my account had hit the max number of cloud formation templates allowed. They weren't being deleted properly because a role + corresponding instance profile cannot be deleted until they're unlinked.

PiperOrigin-RevId: 825126125

Author: hubatish

perfkitbenchmarker/providers/aws/elastic_kubernetes_service.py

@@ -634,8 +634,8 @@ def GetNodeSelectors(self, machine_type: str | None = None) -> list[str]:
 
 
 _KARPENTER_NAMESPACE = 'kube-system'
-_KARPENTER_VERSION = '1.5.0'
-_DEAULT_K8S_VERSION = '1.32'
+_KARPENTER_VERSION = '1.8.1'
+_DEAULT_K8S_VERSION = '1.34'
 
 
 class EksKarpenterCluster(BaseEksCluster):
@@ -670,21 +670,27 @@ def _Create(self):
         '-o',
         template_filename,
     ])
-    vm_util.IssueCommand([
-        'aws',
-        'cloudformation',
-        'deploy',
-        '--stack-name',
-        self.stack_name,
-        '--template-file',
-        template_filename,
-        '--capabilities',
-        'CAPABILITY_NAMED_IAM',
-        '--parameter-overrides',
-        f'ClusterName={self.name}',
-        '--region',
-        f'{self.region}',
-    ])
+    # key=value format differs from other service's Key=key,Value=value format
+    formation_tags = [f'{k}={v}' for k, v in util.MakeDefaultTags().items()]
+    vm_util.IssueCommand(
+        [
+            'aws',
+            'cloudformation',
+            'deploy',
+            '--stack-name',
+            self.stack_name,
+            '--template-file',
+            template_filename,
+            '--capabilities',
+            'CAPABILITY_NAMED_IAM',
+            '--parameter-overrides',
+            f'ClusterName={self.name}',
+            '--region',
+            f'{self.region}',
+            '--tags',
+        ]
+        + formation_tags,
+    )
     create_json: dict[str, Any] = {
         'metadata': {
             'tags': {'karpenter.sh/discovery': self.name},
@@ -1088,15 +1094,56 @@ def _Delete(self):
         self.region,
     ]
     vm_util.IssueCommand(cmd, timeout=1800)
-    vm_util.IssueCommand([
+
+  def _DeleteDependencies(self):
+    """Deletes the CloudFormation stack."""
+    super()._DeleteDependencies()
+    delete_stack_cmd = [
         'aws',
         'cloudformation',
         'delete-stack',
         '--stack-name',
         self.stack_name,
         '--region',
         f'{self.region}',
+    ]
+    # Start deleting the stack but likely to fail to delete this role.
+    vm_util.IssueCommand(delete_stack_cmd)
+    node_role = f'KarpenterNodeRole-{self.name}'
+    out, _, _ = vm_util.IssueCommand([
+        'aws',
+        'iam',
+        'list-instance-profiles-for-role',
+        '--role-name',
+        node_role,
+        '--region',
+        f'{self.region}',
     ])
+    profiles_json = json.loads(out)
+    for profile in profiles_json.get('InstanceProfiles', []):
+      profile_name = profile['InstanceProfileName']
+      vm_util.IssueCommand([
+          'aws',
+          'iam',
+          'remove-role-from-instance-profile',
+          '--instance-profile-name',
+          profile_name,
+          '--role-name',
+          node_role,
+          '--region',
+          f'{self.region}',
+      ])
+      vm_util.IssueCommand([
+          'aws',
+          'iam',
+          'delete-instance-profile',
+          '--instance-profile-name',
+          profile_name,
+          '--region',
+          f'{self.region}',
+      ])
+    # Finish deleting the stack after deleting the role.
+    vm_util.IssueCommand(delete_stack_cmd)
 
   def _IsReady(self):
     """Returns True if cluster is running. Autopilot defaults to 0 nodes."""

tests/providers/aws/elastic_kubernetes_service_test.py

@@ -12,6 +12,7 @@
 from perfkitbenchmarker.providers.aws import aws_network
 from perfkitbenchmarker.providers.aws import elastic_kubernetes_service
 from perfkitbenchmarker.providers.aws import util
+from tests import matchers
 from tests import pkb_common_test_case
 
 
@@ -340,9 +341,13 @@ def testEksYamlCreateFull(self):
     )
     cluster = elastic_kubernetes_service.EksKarpenterCluster(EKS_SPEC)
     self.MockJsonRead(cluster)
-    self.MockIssueCommand(
-        {'create cluster': [('Cluster created', '', 0)], 'curl': [('', '', 0)]}
-    )
+    mock_cmd = self.MockIssueCommand({
+        'cloudformation deploy': [
+            ('Deployed cloud-formation-template.yaml', '', 0)
+        ],
+        'create cluster': [('Cluster created', '', 0)],
+        'curl': [('', '', 0)],
+    })
     cluster._Create()
     assert self.patched_read_json is not None
     called_json = self.patched_read_json.call_args_list[0][0][0]
@@ -376,6 +381,16 @@ def testEksYamlCreateFull(self):
     self.assertEqual(
         called_json['addons'], [{'name': 'eks-pod-identity-agent'}]
     )
+    mock_cmd.func_to_mock.assert_has_calls([
+        mock.call(
+            matchers.HASALLOF(
+                'cloudformation',
+                'deploy',
+                'benchmark=kubernetes_scale',
+                'cloud=aws',
+            )
+        ),
+    ])
 
   def testRecursiveDictionaryUpdate(self):
     base = {'a': 1, 'deep': {'c': 2}}