fix(cli-integ): add retry for iam eventual consistency issue and migration tests for java

packages/@aws-cdk-testing/cli-integ/lib/aws.ts

@@ -276,28 +276,14 @@ export class AwsClients {
   }
 
   public async waitForAssumeRole(roleArn: string) {
-    // Wait until the role has replicated
-    const deadline = Date.now() + 60_000;
-    let lastError: Error | undefined;
-    while (Date.now() < deadline) {
-      try {
-        await this.sts.send(new AssumeRoleCommand({
-          RoleArn: roleArn,
-          RoleSessionName: 'test-existence',
-        }));
-        return;
-      } catch (e: any) {
-        lastError = e;
-
-        if (e.name === 'AccessDenied') {
-          continue;
-        }
-
-        throw e;
-      }
-    }
-
-    throw new Error(`Timed out waiting for role ${roleArn} to become assumable: ${lastError}`);
+    await retryOnMatchingErrors(
+      () => this.sts.send(new AssumeRoleCommand({
+        RoleArn: roleArn,
+        RoleSessionName: 'test-existence',
+      })),
+      ['AccessDenied'],
+      retry.forSeconds(60),
+    );
   }
 
   public async deleteRole(name: string) {
@@ -381,6 +367,36 @@ export async function sleep(ms: number) {
   return new Promise((ok) => setTimeout(ok, ms));
 }
 
+/**
+ * Retry an async operation with error filtering until a deadline is hit.
+ *
+ * Use `retry.forSeconds()` to construct a deadline relative to right now.
+ *
+ * Only retries on errors with matching names in errorNames array.
+ */
+export async function retryOnMatchingErrors<T>(
+  operation: () => Promise<T>,
+  errorNames: string[],
+  deadline: Date,
+  interval: number = 5000,
+): Promise<T> {
+  let i = 0;
+  while (true) {
+    try {
+      i++;
+      return await operation();
+    } catch (e: any) {
+      if (Date.now() > deadline.getTime()) {
+        throw new Error(`Operation did not succeed after ${i} attempts: ${e}`);
+      }
+      if (!errorNames.includes(e.name)) {
+        throw e;
+      }
+      await sleep(interval);
+    }
+  }
+}
+
 function chainableCredentials(region: string): AwsCredentialIdentityProvider {
   if ((process.env.CODEBUILD_BUILD_ARN || process.env.GITHUB_RUN_ID) && process.env.AWS_PROFILE) {
     // in codebuild we must assume the role that the cdk uses

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/cdk-assets-docker-credential.integtest.ts

@@ -5,7 +5,7 @@ import { GetCallerIdentityCommand } from '@aws-sdk/client-sts';
 // eslint-disable-next-line import/no-relative-packages
 import type { DockerDomainCredentialSource } from '../../../../../@aws-cdk/cdk-assets-lib/lib/private/docker-credentials';
 import type { TestFixture } from '../../../lib';
-import { integTest, withDefaultFixture, withRetry } from '../../../lib';
+import { integTest, withDefaultFixture, withRetry, retry } from '../../../lib';
 
 jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime
 
@@ -83,13 +83,17 @@ async function testDockerCredential(fixture: TestFixture, credSource: DockerDoma
   fs.writeFileSync(input, `${domain}\n`);
 
   await fixture.cdkAssets.makeCliAvailable();
-  const output = await fixture.shell(['docker-credential-cdk-assets', 'get'], {
-    modEnv: {
-      ...fixture.cdkShellEnv(),
-      CDK_DOCKER_CREDS_FILE: credsFilePath,
-    },
-    stdio: [fs.openSync(input, 'r')],
-    captureStderr: false,
+  let output: string = '';
+
+  await retry(process.stdout, 'Getting docker credentials', retry.forSeconds(60), async () => {
+    output = await fixture.shell(['docker-credential-cdk-assets', 'get'], {
+      modEnv: {
+        ...fixture.cdkShellEnv(),
+        CDK_DOCKER_CREDS_FILE: credsFilePath,
+      },
+      stdio: [fs.openSync(input, 'r')],
+      captureStderr: false,
+    });
   });
 
   const response = JSON.parse(output);

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/migrate/cdk-migrate-deploys-successfully-java.integtest.ts

@@ -1,13 +1,13 @@
 import { deploysSuccessfully } from './testcase';
-import { integTest, withCDKMigrateFixture } from '../../../lib';
+import { integTest, withCDKMigrateFixture, withRetry } from '../../../lib';
 
 const language = 'java';
 
 jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime
 
 integTest(
   `cdk migrate ${language} deploys successfully`,
-  withCDKMigrateFixture(language, async (fixture) => {
+  withRetry(withCDKMigrateFixture(language, async (fixture) => {
     await deploysSuccessfully(fixture, language);
-  }),
+  })),
 );