fix: add retry for top most frequent failing cli integ tests

packages/@aws-cdk-testing/cli-integ/lib/integ-test.ts

@@ -25,6 +25,35 @@ export interface TestContext {
   reportWaitTime(ms: number): void;
 }
 
+/**
+ * Retry wrapper that executes a test callback up to maxAttempts times
+ *
+ * If any attempt succeeds, it returns immediately. If all attempts fail,
+ * it throws the last error encountered.
+ */
+export function withRetry<T extends TestContext>(
+  callback: (context: T) => Promise<void>,
+  maxAttempts: number = 2,
+): (context: T) => Promise<void> {
+  return async (context: T) => {
+    let lastError;
+
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+      try {
+        await callback(context);
+        return;
+      } catch (error) {
+        lastError = error;
+        if (attempt < maxAttempts) {
+          context.log(`Attempt ${attempt}/${maxAttempts} failed: ${error}. Retrying...`);
+        }
+      }
+    }
+
+    throw lastError;
+  };
+}
+
 /**
  * A wrapper for jest's 'test' which takes regression-disabled tests into account and prints a banner
  */

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/cdk-assets-uses-profile.integtest.ts

@@ -1,89 +1,90 @@
 
 import { promises as fs } from 'fs';
 import * as path from 'path';
-import { integTest, withDefaultFixture } from '../../../lib';
+import { integTest, withDefaultFixture, withRetry } from '../../../lib';
 
 jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime
 
-integTest('cdk-assets uses profile when specified', withDefaultFixture(async (fixture) => {
-  const currentCreds = await fixture.aws.credentials();
+integTest('cdk-assets uses profile when specified',
+  withRetry(withDefaultFixture(async (fixture) => {
+    const currentCreds = await fixture.aws.credentials();
 
-  await fixture.cdkAssets.makeCliAvailable();
+    await fixture.cdkAssets.makeCliAvailable();
 
-  const account = await fixture.aws.account();
-  const region = fixture.aws.region;
-  const bucketName = `cdk-hnb659fds-assets-${account}-${region}`;
+    const account = await fixture.aws.account();
+    const region = fixture.aws.region;
+    const bucketName = `cdk-hnb659fds-assets-${account}-${region}`;
 
-  // Write some asset files. Its important to have more than 1 because cdk-assets
-  // code has some funky state mutations that happens on each asset publishing.
-  const assetFile1 = 'testfile.txt';
-  const assetFile2 = 'testfile.txt';
-  await fs.writeFile(path.join(fixture.integTestDir, assetFile1), 'some asset file');
-  await fs.writeFile(path.join(fixture.integTestDir, assetFile2), 'some asset file');
+    // Write some asset files. Its important to have more than 1 because cdk-assets
+    // code has some funky state mutations that happens on each asset publishing.
+    const assetFile1 = 'testfile.txt';
+    const assetFile2 = 'testfile.txt';
+    await fs.writeFile(path.join(fixture.integTestDir, assetFile1), 'some asset file');
+    await fs.writeFile(path.join(fixture.integTestDir, assetFile2), 'some asset file');
 
-  // Write an asset JSON file to publish to the bootstrapped environment
-  const assetsJson = {
-    version: '38.0.1',
-    files: {
-      testfile1: {
-        source: {
-          path: assetFile1,
-          packaging: 'file',
-        },
-        destinations: {
-          current: {
-            region,
-            assumeRoleArn: `arn:\${AWS::Partition}:iam::${account}:role/cdk-hnb659fds-file-publishing-role-${account}-${region}`,
-            bucketName,
-            objectKey: `test-file1-${Date.now()}.json`,
+    // Write an asset JSON file to publish to the bootstrapped environment
+    const assetsJson = {
+      version: '38.0.1',
+      files: {
+        testfile1: {
+          source: {
+            path: assetFile1,
+            packaging: 'file',
+          },
+          destinations: {
+            current: {
+              region,
+              assumeRoleArn: `arn:\${AWS::Partition}:iam::${account}:role/cdk-hnb659fds-file-publishing-role-${account}-${region}`,
+              bucketName,
+              objectKey: `test-file1-${Date.now()}.json`,
+            },
           },
         },
-      },
-      testfile2: {
-        source: {
-          path: assetFile2,
-          packaging: 'file',
-        },
-        destinations: {
-          current: {
-            region,
-            assumeRoleArn: `arn:\${AWS::Partition}:iam::${account}:role/cdk-hnb659fds-file-publishing-role-${account}-${region}`,
-            bucketName,
-            objectKey: `test-file2-${Date.now()}.json`,
+        testfile2: {
+          source: {
+            path: assetFile2,
+            packaging: 'file',
+          },
+          destinations: {
+            current: {
+              region,
+              assumeRoleArn: `arn:\${AWS::Partition}:iam::${account}:role/cdk-hnb659fds-file-publishing-role-${account}-${region}`,
+              bucketName,
+              objectKey: `test-file2-${Date.now()}.json`,
+            },
           },
         },
       },
-    },
-  };
+    };
 
-  // create a profile with our current credentials.
-  //
-  // if you're wondering why can't we do the reverse (i.e write a bogus profile and assert a failure),
-  // its because when cdk-assets discovers the current account, it DOES consider the profile.
-  // writing a bogus profile would fail this operation and we won't be able to reach the code
-  // we're trying to test.
-  const credentialsFile = path.join(fixture.integTestDir, 'aws.credentials');
-  const profile = 'cdk-assets';
+    // create a profile with our current credentials.
+    //
+    // if you're wondering why can't we do the reverse (i.e write a bogus profile and assert a failure),
+    // its because when cdk-assets discovers the current account, it DOES consider the profile.
+    // writing a bogus profile would fail this operation and we won't be able to reach the code
+    // we're trying to test.
+    const credentialsFile = path.join(fixture.integTestDir, 'aws.credentials');
+    const profile = 'cdk-assets';
 
-  // this kind sucks but its what it is given we need to write a working profile
-  await fs.writeFile(credentialsFile, `[${profile}]
-aws_access_key_id=${currentCreds.accessKeyId}
-aws_secret_access_key=${currentCreds.secretAccessKey}
-aws_session_token=${currentCreds.sessionToken}`);
+    // this kind sucks but its what it is given we need to write a working profile
+    await fs.writeFile(credentialsFile, `[${profile}]
+  aws_access_key_id=${currentCreds.accessKeyId}
+  aws_secret_access_key=${currentCreds.secretAccessKey}
+  aws_session_token=${currentCreds.sessionToken}`);
 
-  await fs.writeFile(path.join(fixture.integTestDir, 'assets.json'), JSON.stringify(assetsJson, undefined, 2));
-  await fixture.shell(['cdk-assets', '--path', 'assets.json', 'publish', '--profile', profile], {
-    modEnv: {
-      ...fixture.cdkShellEnv(),
-      AWS_SHARED_CREDENTIALS_FILE: credentialsFile,
+    await fs.writeFile(path.join(fixture.integTestDir, 'assets.json'), JSON.stringify(assetsJson, undefined, 2));
+    await fixture.shell(['cdk-assets', '--path', 'assets.json', 'publish', '--profile', profile], {
+      modEnv: {
+        ...fixture.cdkShellEnv(),
+        AWS_SHARED_CREDENTIALS_FILE: credentialsFile,
 
-      // remove the default creds so that if the command doesn't use
-      // the profile, it will fail with "Could not load credentials from any providers"
-      AWS_ACCESS_KEY_ID: '',
-      AWS_SECRET_ACCESS_KEY: '',
-      AWS_SESSION_TOKEN: '',
+        // remove the default creds so that if the command doesn't use
+        // the profile, it will fail with "Could not load credentials from any providers"
+        AWS_ACCESS_KEY_ID: '',
+        AWS_SECRET_ACCESS_KEY: '',
+        AWS_SESSION_TOKEN: '',
 
-    },
-  });
-}),
+      },
+    });
+  })),
 );

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/smoketest.integtest.ts

@@ -4,13 +4,13 @@
 import { promises as fs } from 'fs';
 import * as path from 'path';
 import { writeDockerAsset, writeFileAsset } from './asset_helpers';
-import { integTest, withDefaultFixture } from '../../../lib';
+import { integTest, withDefaultFixture, withRetry } from '../../../lib';
 
 jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime
 
 integTest(
   'cdk-assets smoke test',
-  withDefaultFixture(async (fixture) => {
+  withRetry(withDefaultFixture(async (fixture) => {
     await fixture.shell(['npm', 'init', '-y']);
     await fixture.shell(['npm', 'install', 'cdk-assets@latest']);
 
@@ -61,5 +61,5 @@ integTest(
         ...fixture.cdkShellEnv(),
       },
     });
-  }),
+  })),
 );

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/ci-output/cdk-ci-true-output-to-stdout.integtest.ts

@@ -1,11 +1,11 @@
 import type { CdkCliOptions } from '../../../lib';
-import { integTest, withDefaultFixture } from '../../../lib';
+import { integTest, withDefaultFixture, withRetry } from '../../../lib';
 
 jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime
 
 integTest(
   'ci=true output to stdout',
-  withDefaultFixture(async (fixture) => {
+  withRetry(withDefaultFixture(async (fixture) => {
     const execOptions: CdkCliOptions = {
       captureStderr: true,
       onlyStderr: true,
@@ -29,6 +29,6 @@ integTest(
     expect(deployOutput).toEqual('');
     expect(destroyOutput).toEqual('');
     expect(diffOutput).toEqual('');
-  }),
+  })),
 );
 

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib-alpha/cdk-lib-cli-lib-deploy.integtest.ts

@@ -1,11 +1,11 @@
 import { DescribeStackResourcesCommand } from '@aws-sdk/client-cloudformation';
-import { integTest, withCliLibFixture } from '../../../lib';
+import { integTest, withCliLibFixture, withRetry } from '../../../lib';
 
 jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime
 
 integTest(
   'cli-lib deploy',
-  withCliLibFixture(async (fixture) => {
+  withRetry(withCliLibFixture(async (fixture) => {
     const stackName = fixture.fullStackName('simple-1');
 
     try {
@@ -27,6 +27,6 @@ integTest(
         captureStderr: false,
       });
     }
-  }),
+  })),
 );
 

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib-alpha/cdk-lib-cli-lib-list.integtest.ts

@@ -1,12 +1,12 @@
-import { integTest, withCliLibFixture } from '../../../lib';
+import { integTest, withCliLibFixture, withRetry } from '../../../lib';
 
 jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime
 
 integTest(
   'cli-lib list',
-  withCliLibFixture(async (fixture) => {
+  withRetry(withCliLibFixture(async (fixture) => {
     const listing = await fixture.cdk(['list'], { captureStderr: false });
     expect(listing).toContain(fixture.fullStackName('simple-1'));
-  }),
+  })),
 );
 

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib-alpha/cdk-lib-cli-lib-synth.integtest.ts

@@ -1,10 +1,10 @@
-import { integTest, withCliLibFixture } from '../../../lib';
+import { integTest, withCliLibFixture, withRetry } from '../../../lib';
 
 jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime
 
 integTest(
   'cli-lib synth',
-  withCliLibFixture(async (fixture) => {
+  withRetry(withCliLibFixture(async (fixture) => {
     await fixture.cdk(['synth', fixture.fullStackName('simple-1')]);
     expect(fixture.template('simple-1')).toEqual(
       expect.objectContaining({
@@ -22,6 +22,6 @@ integTest(
         }),
       }),
     );
-  }),
+  })),
 );
 

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib-alpha/cdk-lib-security-related-changes-without-a-cli-are-expected-to-fail-when-approval-is-required.integtest.ts

@@ -1,11 +1,11 @@
 import { DescribeStacksCommand } from '@aws-sdk/client-cloudformation';
-import { integTest, withCliLibFixture } from '../../../lib';
+import { integTest, withCliLibFixture, withRetry } from '../../../lib';
 
 jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime
 
 integTest(
   'security related changes without a CLI are expected to fail when approval is required',
-  withCliLibFixture(async (fixture) => {
+  withRetry(withCliLibFixture(async (fixture) => {
     const stdErr = await fixture.cdk(['deploy', fixture.fullStackName('simple-1')], {
       onlyStderr: true,
       captureStderr: true,
@@ -25,5 +25,5 @@ integTest(
         }),
       ),
     ).rejects.toThrow('does not exist');
-  }),
+  })),
 );

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/migrate/cdk-migrate-deploys-successfully-java.integtest.ts

@@ -1,13 +1,13 @@
 import { deploysSuccessfully } from './testcase';
-import { integTest, withCDKMigrateFixture } from '../../../lib';
+import { integTest, withCDKMigrateFixture, withRetry } from '../../../lib';
 
 const language = 'java';
 
 jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime
 
 integTest(
   `cdk migrate ${language} deploys successfully`,
-  withCDKMigrateFixture(language, async (fixture) => {
+  withRetry(withCDKMigrateFixture(language, async (fixture) => {
     await deploysSuccessfully(fixture, language);
-  }),
+  })),
 );