diff --git a/packages/@aws-cdk-testing/cli-integ/lib/integ-test.ts b/packages/@aws-cdk-testing/cli-integ/lib/integ-test.ts index 0247b59fc..27f660548 100644 --- a/packages/@aws-cdk-testing/cli-integ/lib/integ-test.ts +++ b/packages/@aws-cdk-testing/cli-integ/lib/integ-test.ts @@ -25,6 +25,35 @@ export interface TestContext { reportWaitTime(ms: number): void; } +/** + * Retry wrapper that executes a test callback up to maxAttempts times + * + * If any attempt succeeds, it returns immediately. If all attempts fail, + * it throws the last error encountered. + */ +export function withRetry( + callback: (context: T) => Promise, + maxAttempts: number = 2, +): (context: T) => Promise { + return async (context: T) => { + let lastError; + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + await callback(context); + return; + } catch (error) { + lastError = error; + if (attempt < maxAttempts) { + context.log(`Attempt ${attempt}/${maxAttempts} failed: ${error}. Retrying...`); + } + } + } + + throw lastError; + }; +} + /** * A wrapper for jest's 'test' which takes regression-disabled tests into account and prints a banner */ diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/cdk-assets-uses-profile.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/cdk-assets-uses-profile.integtest.ts index 0d50424a6..3bc85e417 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/cdk-assets-uses-profile.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/cdk-assets-uses-profile.integtest.ts @@ -1,89 +1,90 @@ import { promises as fs } from 'fs'; import * as path from 'path'; -import { integTest, withDefaultFixture } from '../../../lib'; +import { integTest, withDefaultFixture, withRetry } from '../../../lib'; jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime -integTest('cdk-assets uses profile when specified', withDefaultFixture(async (fixture) => { - const currentCreds = await fixture.aws.credentials(); +integTest('cdk-assets uses profile when specified', + withRetry(withDefaultFixture(async (fixture) => { + const currentCreds = await fixture.aws.credentials(); - await fixture.cdkAssets.makeCliAvailable(); + await fixture.cdkAssets.makeCliAvailable(); - const account = await fixture.aws.account(); - const region = fixture.aws.region; - const bucketName = `cdk-hnb659fds-assets-${account}-${region}`; + const account = await fixture.aws.account(); + const region = fixture.aws.region; + const bucketName = `cdk-hnb659fds-assets-${account}-${region}`; - // Write some asset files. Its important to have more than 1 because cdk-assets - // code has some funky state mutations that happens on each asset publishing. - const assetFile1 = 'testfile.txt'; - const assetFile2 = 'testfile.txt'; - await fs.writeFile(path.join(fixture.integTestDir, assetFile1), 'some asset file'); - await fs.writeFile(path.join(fixture.integTestDir, assetFile2), 'some asset file'); + // Write some asset files. Its important to have more than 1 because cdk-assets + // code has some funky state mutations that happens on each asset publishing. + const assetFile1 = 'testfile.txt'; + const assetFile2 = 'testfile.txt'; + await fs.writeFile(path.join(fixture.integTestDir, assetFile1), 'some asset file'); + await fs.writeFile(path.join(fixture.integTestDir, assetFile2), 'some asset file'); - // Write an asset JSON file to publish to the bootstrapped environment - const assetsJson = { - version: '38.0.1', - files: { - testfile1: { - source: { - path: assetFile1, - packaging: 'file', - }, - destinations: { - current: { - region, - assumeRoleArn: `arn:\${AWS::Partition}:iam::${account}:role/cdk-hnb659fds-file-publishing-role-${account}-${region}`, - bucketName, - objectKey: `test-file1-${Date.now()}.json`, + // Write an asset JSON file to publish to the bootstrapped environment + const assetsJson = { + version: '38.0.1', + files: { + testfile1: { + source: { + path: assetFile1, + packaging: 'file', + }, + destinations: { + current: { + region, + assumeRoleArn: `arn:\${AWS::Partition}:iam::${account}:role/cdk-hnb659fds-file-publishing-role-${account}-${region}`, + bucketName, + objectKey: `test-file1-${Date.now()}.json`, + }, }, }, - }, - testfile2: { - source: { - path: assetFile2, - packaging: 'file', - }, - destinations: { - current: { - region, - assumeRoleArn: `arn:\${AWS::Partition}:iam::${account}:role/cdk-hnb659fds-file-publishing-role-${account}-${region}`, - bucketName, - objectKey: `test-file2-${Date.now()}.json`, + testfile2: { + source: { + path: assetFile2, + packaging: 'file', + }, + destinations: { + current: { + region, + assumeRoleArn: `arn:\${AWS::Partition}:iam::${account}:role/cdk-hnb659fds-file-publishing-role-${account}-${region}`, + bucketName, + objectKey: `test-file2-${Date.now()}.json`, + }, }, }, }, - }, - }; + }; - // create a profile with our current credentials. - // - // if you're wondering why can't we do the reverse (i.e write a bogus profile and assert a failure), - // its because when cdk-assets discovers the current account, it DOES consider the profile. - // writing a bogus profile would fail this operation and we won't be able to reach the code - // we're trying to test. - const credentialsFile = path.join(fixture.integTestDir, 'aws.credentials'); - const profile = 'cdk-assets'; + // create a profile with our current credentials. + // + // if you're wondering why can't we do the reverse (i.e write a bogus profile and assert a failure), + // its because when cdk-assets discovers the current account, it DOES consider the profile. + // writing a bogus profile would fail this operation and we won't be able to reach the code + // we're trying to test. + const credentialsFile = path.join(fixture.integTestDir, 'aws.credentials'); + const profile = 'cdk-assets'; - // this kind sucks but its what it is given we need to write a working profile - await fs.writeFile(credentialsFile, `[${profile}] -aws_access_key_id=${currentCreds.accessKeyId} -aws_secret_access_key=${currentCreds.secretAccessKey} -aws_session_token=${currentCreds.sessionToken}`); + // this kind sucks but its what it is given we need to write a working profile + await fs.writeFile(credentialsFile, `[${profile}] + aws_access_key_id=${currentCreds.accessKeyId} + aws_secret_access_key=${currentCreds.secretAccessKey} + aws_session_token=${currentCreds.sessionToken}`); - await fs.writeFile(path.join(fixture.integTestDir, 'assets.json'), JSON.stringify(assetsJson, undefined, 2)); - await fixture.shell(['cdk-assets', '--path', 'assets.json', 'publish', '--profile', profile], { - modEnv: { - ...fixture.cdkShellEnv(), - AWS_SHARED_CREDENTIALS_FILE: credentialsFile, + await fs.writeFile(path.join(fixture.integTestDir, 'assets.json'), JSON.stringify(assetsJson, undefined, 2)); + await fixture.shell(['cdk-assets', '--path', 'assets.json', 'publish', '--profile', profile], { + modEnv: { + ...fixture.cdkShellEnv(), + AWS_SHARED_CREDENTIALS_FILE: credentialsFile, - // remove the default creds so that if the command doesn't use - // the profile, it will fail with "Could not load credentials from any providers" - AWS_ACCESS_KEY_ID: '', - AWS_SECRET_ACCESS_KEY: '', - AWS_SESSION_TOKEN: '', + // remove the default creds so that if the command doesn't use + // the profile, it will fail with "Could not load credentials from any providers" + AWS_ACCESS_KEY_ID: '', + AWS_SECRET_ACCESS_KEY: '', + AWS_SESSION_TOKEN: '', - }, - }); -}), + }, + }); + })), ); diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/smoketest.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/smoketest.integtest.ts index be61f7b69..8f1ee95f0 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/smoketest.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/smoketest.integtest.ts @@ -4,13 +4,13 @@ import { promises as fs } from 'fs'; import * as path from 'path'; import { writeDockerAsset, writeFileAsset } from './asset_helpers'; -import { integTest, withDefaultFixture } from '../../../lib'; +import { integTest, withDefaultFixture, withRetry } from '../../../lib'; jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime integTest( 'cdk-assets smoke test', - withDefaultFixture(async (fixture) => { + withRetry(withDefaultFixture(async (fixture) => { await fixture.shell(['npm', 'init', '-y']); await fixture.shell(['npm', 'install', 'cdk-assets@latest']); @@ -61,5 +61,5 @@ integTest( ...fixture.cdkShellEnv(), }, }); - }), + })), ); diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/ci-output/cdk-ci-true-output-to-stdout.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/ci-output/cdk-ci-true-output-to-stdout.integtest.ts index 1e2d6b994..82c3be985 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/ci-output/cdk-ci-true-output-to-stdout.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/ci-output/cdk-ci-true-output-to-stdout.integtest.ts @@ -1,11 +1,11 @@ import type { CdkCliOptions } from '../../../lib'; -import { integTest, withDefaultFixture } from '../../../lib'; +import { integTest, withDefaultFixture, withRetry } from '../../../lib'; jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime integTest( 'ci=true output to stdout', - withDefaultFixture(async (fixture) => { + withRetry(withDefaultFixture(async (fixture) => { const execOptions: CdkCliOptions = { captureStderr: true, onlyStderr: true, @@ -29,6 +29,6 @@ integTest( expect(deployOutput).toEqual(''); expect(destroyOutput).toEqual(''); expect(diffOutput).toEqual(''); - }), + })), ); diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib-alpha/cdk-lib-cli-lib-deploy.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib-alpha/cdk-lib-cli-lib-deploy.integtest.ts index f73081673..111b9b9cc 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib-alpha/cdk-lib-cli-lib-deploy.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib-alpha/cdk-lib-cli-lib-deploy.integtest.ts @@ -1,11 +1,11 @@ import { DescribeStackResourcesCommand } from '@aws-sdk/client-cloudformation'; -import { integTest, withCliLibFixture } from '../../../lib'; +import { integTest, withCliLibFixture, withRetry } from '../../../lib'; jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime integTest( 'cli-lib deploy', - withCliLibFixture(async (fixture) => { + withRetry(withCliLibFixture(async (fixture) => { const stackName = fixture.fullStackName('simple-1'); try { @@ -27,6 +27,6 @@ integTest( captureStderr: false, }); } - }), + })), ); diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib-alpha/cdk-lib-cli-lib-list.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib-alpha/cdk-lib-cli-lib-list.integtest.ts index 561889fa4..474fb7201 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib-alpha/cdk-lib-cli-lib-list.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib-alpha/cdk-lib-cli-lib-list.integtest.ts @@ -1,12 +1,12 @@ -import { integTest, withCliLibFixture } from '../../../lib'; +import { integTest, withCliLibFixture, withRetry } from '../../../lib'; jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime integTest( 'cli-lib list', - withCliLibFixture(async (fixture) => { + withRetry(withCliLibFixture(async (fixture) => { const listing = await fixture.cdk(['list'], { captureStderr: false }); expect(listing).toContain(fixture.fullStackName('simple-1')); - }), + })), ); diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib-alpha/cdk-lib-cli-lib-synth.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib-alpha/cdk-lib-cli-lib-synth.integtest.ts index 2abff1d2a..980526445 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib-alpha/cdk-lib-cli-lib-synth.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib-alpha/cdk-lib-cli-lib-synth.integtest.ts @@ -1,10 +1,10 @@ -import { integTest, withCliLibFixture } from '../../../lib'; +import { integTest, withCliLibFixture, withRetry } from '../../../lib'; jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime integTest( 'cli-lib synth', - withCliLibFixture(async (fixture) => { + withRetry(withCliLibFixture(async (fixture) => { await fixture.cdk(['synth', fixture.fullStackName('simple-1')]); expect(fixture.template('simple-1')).toEqual( expect.objectContaining({ @@ -22,6 +22,6 @@ integTest( }), }), ); - }), + })), ); diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib-alpha/cdk-lib-security-related-changes-without-a-cli-are-expected-to-fail-when-approval-is-required.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib-alpha/cdk-lib-security-related-changes-without-a-cli-are-expected-to-fail-when-approval-is-required.integtest.ts index a7bed6211..98951aab6 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib-alpha/cdk-lib-security-related-changes-without-a-cli-are-expected-to-fail-when-approval-is-required.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib-alpha/cdk-lib-security-related-changes-without-a-cli-are-expected-to-fail-when-approval-is-required.integtest.ts @@ -1,11 +1,11 @@ import { DescribeStacksCommand } from '@aws-sdk/client-cloudformation'; -import { integTest, withCliLibFixture } from '../../../lib'; +import { integTest, withCliLibFixture, withRetry } from '../../../lib'; jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime integTest( 'security related changes without a CLI are expected to fail when approval is required', - withCliLibFixture(async (fixture) => { + withRetry(withCliLibFixture(async (fixture) => { const stdErr = await fixture.cdk(['deploy', fixture.fullStackName('simple-1')], { onlyStderr: true, captureStderr: true, @@ -25,5 +25,5 @@ integTest( }), ), ).rejects.toThrow('does not exist'); - }), + })), ); diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/migrate/cdk-migrate-deploys-successfully-java.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/migrate/cdk-migrate-deploys-successfully-java.integtest.ts index 258f8f970..618953391 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/migrate/cdk-migrate-deploys-successfully-java.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/migrate/cdk-migrate-deploys-successfully-java.integtest.ts @@ -1,5 +1,5 @@ import { deploysSuccessfully } from './testcase'; -import { integTest, withCDKMigrateFixture } from '../../../lib'; +import { integTest, withCDKMigrateFixture, withRetry } from '../../../lib'; const language = 'java'; @@ -7,7 +7,7 @@ jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-c integTest( `cdk migrate ${language} deploys successfully`, - withCDKMigrateFixture(language, async (fixture) => { + withRetry(withCDKMigrateFixture(language, async (fixture) => { await deploysSuccessfully(fixture, language); - }), + })), );