fix(integ-testing): add test that forces cdk-assets to use the Docker credentials helper (#663)

This runs `docker-credential-cdk-asset` (the Docker helper tool) and
checks its output to see that it actually usefully does something.

- I would have liked to do a full integration between `cdk-assets` and
the correct Docker config, but unfortunately this only makes sense for a
2nd registry (not the primary registry we're pushing to). So we would
need either:
- A fake docker registry; I tried looking for one or faking one via a
static HTTP server, but the requirement of signing the image manifest
got me stuck. Also, the Docker registry/HTTP server needs to be
accessible from the Docker Host, so it can't easily run inside the
CodeBuild/GHA container; we would need to set up AWS infra to host this
static server which implies S3 + CloudFront or APIGW + Lambda, which
starts to get annoying.
- A real Docker registry that requires user/pass authentication; but now
we have to manage a username and password and security is going to make
us rotate it.
- An ECR repository; it must be in a different account or region,
because the registry name (`<account>.dkr.ecr.<region>.amazonaws.com`)
must be different. To make it in a different account or region, we would
need to run Atmosphere twice, but that doesn't help when we're running
locally or in a non-Atmosphere enabled environment.

- I would also have liked to do a test that reads from SecretsManager,
but at least the Atmosphere roles don't have permissions to use
SecretsManager and I'm not sure what other environments these tests run
in, so this test is commented out for now.

## Also in this PR

- Features to create and clean up a role in the integ tests.
- Properly configure the `cdk-assets` version to test (not just
`latest`, that will work in our online builds with a fake NPM repo but
won't work locally, for example). This needed some refactoring of the
"package sources" code to generalize the classes that were dealing with
the `aws-cdk` package to take a package name.
- Refactored the assets helper code a bit to pull out some helper
functions used to prepare assets.
- Fix a bug in `yarn-cling` that allows hoisting conflicting
dependencies into certain places if `yarn` has already hoisted them
during the proper install.


---
By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache-2.0 license

---------

Signed-off-by: github-actions <github-actions@github.com>
Co-authored-by: github-actions <github-actions@github.com>

Author: rix0rrr

.projenrc.ts

@@ -1639,14 +1639,15 @@ const cliInteg = configureProject(
       '@aws-sdk/client-sns',
       '@aws-sdk/client-sso',
       '@aws-sdk/client-sts',
+      '@aws-sdk/client-secrets-manager',
       '@aws-sdk/credential-providers',
       '@cdklabs/cdk-atmosphere-client',
       '@smithy/util-retry', // smithy packages don't have the same major version as SDK packages
       '@smithy/types', // smithy packages don't have the same major version as SDK packages
       'axios@^1',
       'chalk@^4',
       'fs-extra@^9',
-      'glob@^7',
+      'glob@^9',
       'make-runnable@^1',
       'mockttp@^3',
       'npm@^10',
@@ -1660,6 +1661,7 @@ const cliInteg = configureProject(
       'jest@^29',
       'jest-junit@^15',
       'ts-jest@^29',
+      'proxy-agent',
       'node-pty',
     ],
     devDeps: [
@@ -1668,7 +1670,6 @@ const cliInteg = configureProject(
       '@types/semver@^7',
       '@types/yargs@^16',
       '@types/fs-extra@^9',
-      '@types/glob@^7',
     ],
     bin: {
       'run-suite': 'bin/run-suite',

packages/@aws-cdk-testing/cli-integ/.projen/deps.json

@@ -8,7 +8,7 @@ import {
 import { DeleteRepositoryCommand, ECRClient } from '@aws-sdk/client-ecr';
 import { ECRPUBLICClient } from '@aws-sdk/client-ecr-public';
 import { ECSClient } from '@aws-sdk/client-ecs';
-import { IAMClient } from '@aws-sdk/client-iam';
+import { CreateRoleCommand, DeleteRoleCommand, DeleteRolePolicyCommand, IAMClient, ListRolePoliciesCommand, PutRolePolicyCommand } from '@aws-sdk/client-iam';
 import { LambdaClient } from '@aws-sdk/client-lambda';
 import {
   S3Client,
@@ -17,27 +17,31 @@ import {
   type ObjectIdentifier,
   DeleteBucketCommand,
 } from '@aws-sdk/client-s3';
+import { SecretsManagerClient } from '@aws-sdk/client-secrets-manager';
 import { SNSClient } from '@aws-sdk/client-sns';
 import { SSOClient } from '@aws-sdk/client-sso';
-import { STSClient, GetCallerIdentityCommand } from '@aws-sdk/client-sts';
+import { AssumeRoleCommand, STSClient, GetCallerIdentityCommand } from '@aws-sdk/client-sts';
 import { fromIni, fromNodeProviderChain } from '@aws-sdk/credential-providers';
-import type { AwsCredentialIdentity, AwsCredentialIdentityProvider } from '@smithy/types';
+import type { AwsCredentialIdentity, AwsCredentialIdentityProvider, NodeHttpHandlerOptions } from '@smithy/types';
 import { ConfiguredRetryStrategy } from '@smithy/util-retry';
+
 interface ClientConfig {
   readonly credentials: AwsCredentialIdentityProvider | AwsCredentialIdentity;
   readonly region: string;
   readonly retryStrategy: ConfiguredRetryStrategy;
+  readonly requestHandler?: NodeHttpHandlerOptions;
 }
 
 export class AwsClients {
-  public static async forIdentity(region: string, identity: AwsCredentialIdentity, output: NodeJS.WritableStream) {
-    return new AwsClients(region, output, identity);
+  public static async forIdentity(randomString: string, region: string, identity: AwsCredentialIdentity, output: NodeJS.WritableStream) {
+    return new AwsClients(randomString, region, output, identity);
   }
 
-  public static async forRegion(region: string, output: NodeJS.WritableStream) {
-    return new AwsClients(region, output);
+  public static async forRegion(randomString: string, region: string, output: NodeJS.WritableStream) {
+    return new AwsClients(randomString, region, output);
   }
 
+  private readonly cleanup: (() => Promise<void>)[] = [];
   private readonly config: ClientConfig;
 
   public readonly cloudFormation: CloudFormationClient;
@@ -50,8 +54,11 @@ export class AwsClients {
   public readonly iam: IAMClient;
   public readonly lambda: LambdaClient;
   public readonly sts: STSClient;
+  public readonly secretsManager: SecretsManagerClient;
 
-  constructor(
+  private constructor(
+    /** A random string to use for temporary resources, like roles (should preferably match unique test-specific randomString) */
+    private readonly randomString: string,
     public readonly region: string,
     private readonly output: NodeJS.WritableStream,
     public readonly identity?: AwsCredentialIdentity) {
@@ -60,6 +67,7 @@ export class AwsClients {
       region: this.region,
       retryStrategy: new ConfiguredRetryStrategy(9, (attempt: number) => attempt ** 500),
     };
+
     this.cloudFormation = new CloudFormationClient(this.config);
     this.s3 = new S3Client(this.config);
     this.ecr = new ECRClient(this.config);
@@ -70,6 +78,22 @@ export class AwsClients {
     this.iam = new IAMClient(this.config);
     this.lambda = new LambdaClient(this.config);
     this.sts = new STSClient(this.config);
+    this.secretsManager = new SecretsManagerClient(this.config);
+  }
+
+  public addCleanup(cleanup: () => Promise<any>) {
+    this.cleanup.push(cleanup);
+  }
+
+  public async dispose() {
+    for (const cleanup of this.cleanup) {
+      try {
+        await cleanup();
+      } catch (e: any) {
+        this.output.write(`⚠️ Error during cleanup: ${e.message}\n`);
+      }
+    }
+    this.cleanup.splice(0, this.cleanup.length);
   }
 
   public async account(): Promise<string> {
@@ -219,6 +243,79 @@ export class AwsClients {
       throw e;
     }
   }
+
+  /**
+   * Create a role that will be cleaned up when the AwsClients object is cleaned up
+   */
+  public async temporaryRole(namePrefix: string, assumeRolePolicyStatements: any[], policyStatements: any[]) {
+    const response = await this.iam.send(new CreateRoleCommand({
+      RoleName: `${namePrefix}-${this.randomString}`,
+      AssumeRolePolicyDocument: JSON.stringify({
+        Version: '2012-10-17',
+        Statement: assumeRolePolicyStatements,
+      }, undefined, 2),
+      Tags: [
+        {
+          Key: 'deleteme',
+          Value: 'true',
+        },
+      ],
+    }));
+    await this.iam.send(new PutRolePolicyCommand({
+      RoleName: `${namePrefix}-${this.randomString}`,
+      PolicyName: 'DefaultPolicy',
+      PolicyDocument: JSON.stringify({
+        Version: '2012-10-17',
+        Statement: policyStatements,
+      }, undefined, 2),
+    }));
+
+    this.addCleanup(() => this.deleteRole(response.Role!.RoleName!));
+
+    return response.Role?.Arn ?? '*CreateRole did not return an ARN*';
+  }
+
+  public async waitForAssumeRole(roleArn: string) {
+    // Wait until the role has replicated
+    const deadline = Date.now() + 60_000;
+    let lastError: Error | undefined;
+    while (Date.now() < deadline) {
+      try {
+        await this.sts.send(new AssumeRoleCommand({
+          RoleArn: roleArn,
+          RoleSessionName: 'test-existence',
+        }));
+        return;
+      } catch (e: any) {
+        lastError = e;
+
+        if (e.name === 'AccessDenied') {
+          continue;
+        }
+
+        throw e;
+      }
+    }
+
+    throw new Error(`Timed out waiting for role ${roleArn} to become assumable: ${lastError}`);
+  }
+
+  public async deleteRole(name: string) {
+    const policiesResponse = await this.iam.send(new ListRolePoliciesCommand({
+      RoleName: name,
+    }));
+
+    for (const policyName of policiesResponse.PolicyNames ?? []) {
+      await this.iam.send(new DeleteRolePolicyCommand({
+        RoleName: name,
+        PolicyName: policyName,
+      }));
+    }
+
+    await this.iam.send(new DeleteRoleCommand({
+      RoleName: name,
+    }));
+  }
 }
 
 export function isStackMissingError(e: Error) {

packages/@aws-cdk-testing/cli-integ/.projen/tasks.json

@@ -11,6 +11,9 @@ import { RunnerLibraryPreinstalledSource } from '../package-sources/library-prei
 import type { IRunnerSource, ITestCliSource, ITestLibrarySource } from '../package-sources/source';
 import { serializeSources } from '../package-sources/subprocess';
 
+const CLI_PACKAGE_NAME = 'aws-cdk';
+const CDK_ASSETS_PACKAGE_NAME = 'cdk-assets';
+
 async function main() {
   const args = await yargs
     .command('* <SUITENAME>', 'default command', y => y
@@ -41,6 +44,11 @@ async function main() {
         alias: 'l',
         type: 'string',
       })
+      .options('cdk-assets-version', {
+        describe: 'cdk-assets version to use.',
+        alias: 'a',
+        type: 'string',
+      })
       .option('use-source', {
         descripton: 'Use TypeScript packages from the given source repository (or "auto")',
         type: 'string',
@@ -105,25 +113,55 @@ async function main() {
   const suiteName = args.SUITENAME;
 
   // So many ways to specify this, and with various ways to spell the same flag (o_O)
+  // Also, some of them depend on each other for convenience.
   const cliSource = new UniqueOption<IRunnerSource<ITestCliSource>>('CLI version');
+  const cdkAssetsSource = new UniqueOption<IRunnerSource<ITestCliSource>>('cdk-assets version');
+
+  // Specific CLI version
   for (const flagAlias of ['cli-version', 'use-cli-release'] as const) {
     if (args[flagAlias]) {
-      cliSource.set(new RunnerCliNpmSource(args[flagAlias]), `--${flagAlias}`);
+      cliSource.set(new RunnerCliNpmSource(CLI_PACKAGE_NAME, args[flagAlias]), `--${flagAlias}`);
     }
   }
+
+  // Specific cdk-assets version
+  if (args['cdk-assets-version']) {
+    cdkAssetsSource.set(new RunnerCliNpmSource(CDK_ASSETS_PACKAGE_NAME, args['cdk-assets-version']), '--cdk-assets-version');
+  }
+
+  // Specifically use a source location
   for (const flagAlias of ['cli-source', 'use-source'] as const) {
     if (args[flagAlias]) {
       const root = args[flagAlias] === 'auto' ? await autoFindRepoRoot() : args[flagAlias];
-      cliSource.set(new RunnerCliRepoSource(root), `--${flagAlias}`);
+      cliSource.set(new RunnerCliRepoSource(CLI_PACKAGE_NAME, root), `--${flagAlias}`);
+      cdkAssetsSource.set(new RunnerCliRepoSource(CDK_ASSETS_PACKAGE_NAME, root), `--${flagAlias}`);
     }
   }
+
+  // Specifically request that a source location is given, or we didn't find a CLI yet.
+  // A CLI source is required, so if this fails that's alright.
   if (args['auto-source'] || !cliSource.isSet()) {
-    cliSource.set(new RunnerCliRepoSource(await autoFindRepoRoot()), '--auto-source');
+    cliSource.set(new RunnerCliRepoSource(CLI_PACKAGE_NAME, await autoFindRepoRoot()), '--auto-source');
+  }
+
+  // If the CLI is taken from the source, and cdk-assets is not set, we can copy the cdk-assets source from the CLI source.
+  if (!cdkAssetsSource.isSet()) {
+    const cliSrc = cliSource.assert();
+    if (cliSrc instanceof RunnerCliRepoSource) {
+      cdkAssetsSource.set(new RunnerCliRepoSource(CDK_ASSETS_PACKAGE_NAME, cliSrc.repoRoot), 'copied from CLI source');
+    }
   }
 
+  // If cdk-assets is still not configured, fall back to the latest version that is available
+  if (!cdkAssetsSource.isSet()) {
+    cdkAssetsSource.set(new RunnerCliNpmSource(CDK_ASSETS_PACKAGE_NAME, 'latest'), '--cdk-assets-version not set');
+  }
+
+  // Library source is either the given one, or 'latest' (nice and simple)
   const librarySource: IRunnerSource<ITestLibrarySource>
     = new RunnerLibraryNpmSource('aws-cdk-lib', args['framework-version'] ? args['framework-version'] : 'latest');
 
+  // Toolkit lib source is either the given one, or the one that's being brought by 'package.json' already, or 'latest'
   const toolkitLibPackage = '@aws-cdk/toolkit-lib';
   let toolkitSource: IRunnerSource<ITestLibrarySource> | undefined;
   if (args['toolkit-lib-version']) {
@@ -141,6 +179,7 @@ async function main() {
   console.log(`        CLI source:         ${cliSource.assert().sourceDescription}`);
   console.log(`        Library source:     ${librarySource.sourceDescription}`);
   console.log(`        Toolkit lib source: ${toolkitSource.sourceDescription}`);
+  console.log(`        cdk-assets source:  ${cdkAssetsSource.assert().sourceDescription}`);
 
   if (args.verbose) {
     process.env.VERBOSE = '1';
@@ -171,10 +210,15 @@ async function main() {
     disposables.push(toolkitLib);
     console.log(`        Toolkit library: ${toolkitLib.version}`);
 
+    const cdkAssets = await cdkAssetsSource.assert().runnerPrepare();
+    disposables.push(cdkAssets);
+    console.log(`        cdk-assets:      ${cdkAssets.version}`);
+
     serializeSources({
       cli,
       library,
       toolkitLib,
+      cdkAssets,
     });
 
     const jestConfig = path.resolve(__dirname, '..', '..', 'resources', 'integ.jest.config.js');

packages/@aws-cdk-testing/cli-integ/lib/aws.ts

@@ -7,6 +7,7 @@ const MINIMUM_VERSION = '3.9';
 export async function npmMostRecentMatching(packageName: string, range: string) {
   const output = JSON.parse(await shell(['node', require.resolve('npm'), '--silent', 'view', `${packageName}@${range}`, 'version', '--json'], {
     show: 'error',
+    captureStderr: false,
   }));
 
   if (typeof output === 'string') {

packages/@aws-cdk-testing/cli-integ/lib/cli/run-suite.ts

@@ -8,20 +8,20 @@ import { addToShellPath, rimraf, shell } from '../shell';
 export class RunnerCliNpmSource implements IRunnerSource<ITestCliSource> {
   public readonly sourceDescription: string;
 
-  constructor(private readonly range: string) {
+  constructor(private readonly packageName: string, private readonly range: string) {
     this.sourceDescription = `${this.range} (npm)`;
   }
 
   public async runnerPrepare(): Promise<IPreparedRunnerSource<ITestCliSource>> {
     const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'tmpcdk'));
     fs.mkdirSync(tempDir, { recursive: true });
 
-    await shell(['node', require.resolve('npm'), 'install', `aws-cdk@${this.range}`], {
+    await shell(['node', require.resolve('npm'), 'install', `${this.packageName}@${this.range}`], {
       cwd: tempDir,
       show: 'error',
       outputs: [process.stderr],
     });
-    const installedVersion = await npmQueryInstalledVersion('aws-cdk', tempDir);
+    const installedVersion = await npmQueryInstalledVersion(this.packageName, tempDir);
 
     return {
       version: installedVersion,