diff --git a/packages/@aws-cdk-testing/cli-integ/lib/aws.ts b/packages/@aws-cdk-testing/cli-integ/lib/aws.ts index 301de7f8b..79d682e5b 100644 --- a/packages/@aws-cdk-testing/cli-integ/lib/aws.ts +++ b/packages/@aws-cdk-testing/cli-integ/lib/aws.ts @@ -276,28 +276,14 @@ export class AwsClients { } public async waitForAssumeRole(roleArn: string) { - // Wait until the role has replicated - const deadline = Date.now() + 60_000; - let lastError: Error | undefined; - while (Date.now() < deadline) { - try { - await this.sts.send(new AssumeRoleCommand({ - RoleArn: roleArn, - RoleSessionName: 'test-existence', - })); - return; - } catch (e: any) { - lastError = e; - - if (e.name === 'AccessDenied') { - continue; - } - - throw e; - } - } - - throw new Error(`Timed out waiting for role ${roleArn} to become assumable: ${lastError}`); + await retryOnMatchingErrors( + () => this.sts.send(new AssumeRoleCommand({ + RoleArn: roleArn, + RoleSessionName: 'test-existence', + })), + ['AccessDenied'], + retry.forSeconds(60), + ); } public async deleteRole(name: string) { @@ -381,6 +367,36 @@ export async function sleep(ms: number) { return new Promise((ok) => setTimeout(ok, ms)); } +/** + * Retry an async operation with error filtering until a deadline is hit. + * + * Use `retry.forSeconds()` to construct a deadline relative to right now. + * + * Only retries on errors with matching names in errorNames array. + */ +export async function retryOnMatchingErrors( + operation: () => Promise, + errorNames: string[], + deadline: Date, + interval: number = 5000, +): Promise { + let i = 0; + while (true) { + try { + i++; + return await operation(); + } catch (e: any) { + if (Date.now() > deadline.getTime()) { + throw new Error(`Operation did not succeed after ${i} attempts: ${e}`); + } + if (!errorNames.includes(e.name)) { + throw e; + } + await sleep(interval); + } + } +} + function chainableCredentials(region: string): AwsCredentialIdentityProvider { if ((process.env.CODEBUILD_BUILD_ARN || process.env.GITHUB_RUN_ID) && process.env.AWS_PROFILE) { // in codebuild we must assume the role that the cdk uses diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/cdk-assets-docker-credential.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/cdk-assets-docker-credential.integtest.ts index 7ea4555a2..b1c1c45ea 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/cdk-assets-docker-credential.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/cdk-assets-docker-credential.integtest.ts @@ -5,7 +5,7 @@ import { GetCallerIdentityCommand } from '@aws-sdk/client-sts'; // eslint-disable-next-line import/no-relative-packages import type { DockerDomainCredentialSource } from '../../../../../@aws-cdk/cdk-assets-lib/lib/private/docker-credentials'; import type { TestFixture } from '../../../lib'; -import { integTest, withDefaultFixture, withRetry } from '../../../lib'; +import { integTest, withDefaultFixture, withRetry, retry } from '../../../lib'; jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime @@ -83,13 +83,17 @@ async function testDockerCredential(fixture: TestFixture, credSource: DockerDoma fs.writeFileSync(input, `${domain}\n`); await fixture.cdkAssets.makeCliAvailable(); - const output = await fixture.shell(['docker-credential-cdk-assets', 'get'], { - modEnv: { - ...fixture.cdkShellEnv(), - CDK_DOCKER_CREDS_FILE: credsFilePath, - }, - stdio: [fs.openSync(input, 'r')], - captureStderr: false, + let output: string = ''; + + await retry(process.stdout, 'Getting docker credentials', retry.forSeconds(60), async () => { + output = await fixture.shell(['docker-credential-cdk-assets', 'get'], { + modEnv: { + ...fixture.cdkShellEnv(), + CDK_DOCKER_CREDS_FILE: credsFilePath, + }, + stdio: [fs.openSync(input, 'r')], + captureStderr: false, + }); }); const response = JSON.parse(output); diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/migrate/cdk-migrate-deploys-successfully-java.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/migrate/cdk-migrate-deploys-successfully-java.integtest.ts index 258f8f970..618953391 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/migrate/cdk-migrate-deploys-successfully-java.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/migrate/cdk-migrate-deploys-successfully-java.integtest.ts @@ -1,5 +1,5 @@ import { deploysSuccessfully } from './testcase'; -import { integTest, withCDKMigrateFixture } from '../../../lib'; +import { integTest, withCDKMigrateFixture, withRetry } from '../../../lib'; const language = 'java'; @@ -7,7 +7,7 @@ jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-c integTest( `cdk migrate ${language} deploys successfully`, - withCDKMigrateFixture(language, async (fixture) => { + withRetry(withCDKMigrateFixture(language, async (fixture) => { await deploysSuccessfully(fixture, language); - }), + })), );