feat: add retry mechanism for flaky cli integ tests (#753)

We have observed frequent pipeline build failures due to flaky
integration tests that usually succeed on retries. Currently, these
tests have to be retried manually for unblocking the pipelines. By
enabling automated retries for these flaky tests, we can reduce
intermittent pipeline failures and eliminate unnecessary manual
intervention which then will only be required for test failures that
persist even after retries.

The change adds a retry wrapper for specific CLI integration tests that
have been identified as frequently failing but succeeding on retries.
The goal is to identify the the frequently failing flaky test that
require manual intervention and add retry mechanism before we
investigate the tests individually.

Fixes : cdklabs/cdk-ops#4176

### Top failing tests from last week
<img width="1303" height="207" alt="image"
src="https://github.com/user-attachments/assets/9911effe-a422-4327-a602-bce2b0c77121"
/>

  ---

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache-2.0 license

Author: abidhasan-aws

packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts

@@ -163,6 +163,36 @@ export function withCDKMigrateFixture(language: string, block: (content: TestFix
   return withAws(withTimeout(DEFAULT_TEST_TIMEOUT_S, withCdkMigrateApp(language, block)));
 }
 
+/**
+ * Retry wrapper that executes a test callback up to maxAttempts times
+ *
+ * If any attempt succeeds, it returns immediately. If all attempts fail,
+ * it throws the last error encountered.
+ */
+
+export function withRetry<T extends TestContext>(
+  callback: (context: T) => Promise<void>,
+  maxAttempts: number = 2,
+): (context: T) => Promise<void> {
+  return async (context: T) => {
+    let lastError;
+
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+      try {
+        await callback(context);
+        return;
+      } catch (error) {
+        lastError = error;
+        if (attempt < maxAttempts) {
+          context.log(`Attempt ${attempt}/${maxAttempts} failed: ${error}. Retrying...`);
+        }
+      }
+    }
+
+    throw lastError;
+  };
+}
+
 export interface DisableBootstrapContext {
   /**
    * Whether to disable creating the default bootstrap

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/cdk-assets-docker-credential.integtest.ts

@@ -5,13 +5,13 @@ import { GetCallerIdentityCommand } from '@aws-sdk/client-sts';
 // eslint-disable-next-line import/no-relative-packages
 import type { DockerDomainCredentialSource } from '../../../../../@aws-cdk/cdk-assets-lib/lib/private/docker-credentials';
 import type { TestFixture } from '../../../lib';
-import { integTest, withDefaultFixture } from '../../../lib';
+import { integTest, withDefaultFixture, withRetry } from '../../../lib';
 
 jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime
 
 integTest(
   'docker-credential-cdk-assets can assume role and fetch ECR credentials',
-  withDefaultFixture(async (fixture) => {
+  withRetry(withDefaultFixture(async (fixture) => {
     const caller = await fixture.aws.sts.send(new GetCallerIdentityCommand({}));
 
     const roleArn = await fixture.aws.temporaryRole('ecr-repo-role', [
@@ -35,7 +35,7 @@ integTest(
       // This role must have permissions to call `ecr:GetAuthorizationToken`
       assumeRoleArn: roleArn,
     });
-  }),
+  })),
 );
 
 /*