fix(cli-integ): add retry for iam eventual consistency issue and migration tests for java (#788)

### Background
The CDK pipelines have been experiencing intermittent failures due to
flaky tests that typically pass on retry. This pull request addresses
the investigation of the two most frequent failing tests.

<img width="1307" height="530" alt="image (1)"
src="https://github.com/user-attachments/assets/c03da25a-6921-4358-8a12-81db8722d437"
/>


### AWS IAM Eventual Consistency Issue
Test: `docker-credential-cdk-assets can assume role and fetch ECR
credentials`

Issue: Docker credential fetching fails with AccessDenied errors because
newly created IAM roles and policies require time to propagate across
AWS regions.

Fix: Implemented a 60-second retry mechanism for
`fetchDockerLoginCredentials()` when encountering AccessDenied errors.

### CDK Migration Test Instability
Test: `cdk migrate java deploys successfully`

Issue: Java CDK migration tests fail sporadically due to Maven Central
repository rate limiting errors & dependency resolution failure

Fix: Implemented full test retry logic as these transient
network-related issues could not be reproduced in local environments.

### Impact
These changes should improve pipeline stability and reduce the need for
manual intervention.


---
By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache-2.0 license

---------

Signed-off-by: github-actions <[email protected]>
Co-authored-by: github-actions <[email protected]>

Author: abidhasan-aws

packages/@aws-cdk-testing/cli-integ/lib/aws.ts

@@ -276,28 +276,14 @@ export class AwsClients {
   }
 
   public async waitForAssumeRole(roleArn: string) {
-    // Wait until the role has replicated
-    const deadline = Date.now() + 60_000;
-    let lastError: Error | undefined;
-    while (Date.now() < deadline) {
-      try {
-        await this.sts.send(new AssumeRoleCommand({
-          RoleArn: roleArn,
-          RoleSessionName: 'test-existence',
-        }));
-        return;
-      } catch (e: any) {
-        lastError = e;
-
-        if (e.name === 'AccessDenied') {
-          continue;
-        }
-
-        throw e;
-      }
-    }
-
-    throw new Error(`Timed out waiting for role ${roleArn} to become assumable: ${lastError}`);
+    await retryOnMatchingErrors(
+      () => this.sts.send(new AssumeRoleCommand({
+        RoleArn: roleArn,
+        RoleSessionName: 'test-existence',
+      })),
+      ['AccessDenied'],
+      retry.forSeconds(60),
+    );
   }
 
   public async deleteRole(name: string) {
@@ -381,6 +367,36 @@ export async function sleep(ms: number) {
   return new Promise((ok) => setTimeout(ok, ms));
 }
 
+/**
+ * Retry an async operation with error filtering until a deadline is hit.
+ *
+ * Use `retry.forSeconds()` to construct a deadline relative to right now.
+ *
+ * Only retries on errors with matching names in errorNames array.
+ */
+export async function retryOnMatchingErrors<T>(
+  operation: () => Promise<T>,
+  errorNames: string[],
+  deadline: Date,
+  interval: number = 5000,
+): Promise<T> {
+  let i = 0;
+  while (true) {
+    try {
+      i++;
+      return await operation();
+    } catch (e: any) {
+      if (Date.now() > deadline.getTime()) {
+        throw new Error(`Operation did not succeed after ${i} attempts: ${e}`);
+      }
+      if (!errorNames.includes(e.name)) {
+        throw e;
+      }
+      await sleep(interval);
+    }
+  }
+}
+
 function chainableCredentials(region: string): AwsCredentialIdentityProvider {
   if ((process.env.CODEBUILD_BUILD_ARN || process.env.GITHUB_RUN_ID) && process.env.AWS_PROFILE) {
     // in codebuild we must assume the role that the cdk uses

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/cdk-assets-docker-credential.integtest.ts

@@ -5,7 +5,7 @@ import { GetCallerIdentityCommand } from '@aws-sdk/client-sts';
 // eslint-disable-next-line import/no-relative-packages
 import type { DockerDomainCredentialSource } from '../../../../../@aws-cdk/cdk-assets-lib/lib/private/docker-credentials';
 import type { TestFixture } from '../../../lib';
-import { integTest, withDefaultFixture, withRetry } from '../../../lib';
+import { integTest, withDefaultFixture, withRetry, retry } from '../../../lib';
 
 jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime
 
@@ -83,13 +83,17 @@ async function testDockerCredential(fixture: TestFixture, credSource: DockerDoma
   fs.writeFileSync(input, `${domain}\n`);
 
   await fixture.cdkAssets.makeCliAvailable();
-  const output = await fixture.shell(['docker-credential-cdk-assets', 'get'], {
-    modEnv: {
-      ...fixture.cdkShellEnv(),
-      CDK_DOCKER_CREDS_FILE: credsFilePath,
-    },
-    stdio: [fs.openSync(input, 'r')],
-    captureStderr: false,
+  let output: string = '';
+
+  await retry(process.stdout, 'Getting docker credentials', retry.forSeconds(60), async () => {
+    output = await fixture.shell(['docker-credential-cdk-assets', 'get'], {
+      modEnv: {
+        ...fixture.cdkShellEnv(),
+        CDK_DOCKER_CREDS_FILE: credsFilePath,
+      },
+      stdio: [fs.openSync(input, 'r')],
+      captureStderr: false,
+    });
   });
 
   const response = JSON.parse(output);

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/migrate/cdk-migrate-deploys-successfully-java.integtest.ts

@@ -1,13 +1,13 @@
 import { deploysSuccessfully } from './testcase';
-import { integTest, withCDKMigrateFixture } from '../../../lib';
+import { integTest, withCDKMigrateFixture, withRetry } from '../../../lib';
 
 const language = 'java';
 
 jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime
 
 integTest(
   `cdk migrate ${language} deploys successfully`,
-  withCDKMigrateFixture(language, async (fixture) => {
+  withRetry(withCDKMigrateFixture(language, async (fixture) => {
     await deploysSuccessfully(fixture, language);
-  }),
+  })),
 );