feat(integ-runner): detect bootstrap errors and retry tests in valid regions

Add automatic detection of bootstrap-related failures during integration
test runs. When a region is not bootstrapped, the integ-runner now:

- Detects bootstrap errors using a strongly-typed BootstrapError class
- Removes the non-bootstrapped region from the environment pool
- Re-queues failed tests to run in remaining valid regions
- Provides clear feedback with the exact `cdk bootstrap` command needed

This prevents wasted CI time from repeatedly scheduling tests in
non-bootstrapped regions and gives users actionable error messages.

Author: mrgrain

.projenrc.ts

@@ -1447,6 +1447,7 @@ const integRunner = configureProject(
       '@types/yargs',
       'constructs@^10',
       '@aws-cdk/[email protected]',
+      'fast-check@^3.23.2',
     ],
     allowPrivateDeps: true,
     tsconfig: {

packages/@aws-cdk/integ-runner/.projen/deps.json

@@ -8,7 +8,7 @@ import type { IntegTest, IntegTestInfo } from './runner/integration-tests';
 import { IntegrationTests } from './runner/integration-tests';
 import { processUnstableFeatures, availableFeaturesDescription } from './unstable-features';
 import type { IntegRunnerMetrics, IntegTestWorkerConfig, DestructiveChange } from './workers';
-import { runSnapshotTests, runIntegrationTests } from './workers';
+import { runSnapshotTests, runIntegrationTests, printRemovedEnvironmentsSummary } from './workers';
 import { watchIntegrationTest } from './workers/integ-watch-worker';
 
 // https://github.com/yargs/yargs/issues/1929
@@ -184,7 +184,7 @@ async function run(options: ReturnType<typeof parseCliArgs>) {
 
     // run integration tests if `--update-on-failed` OR `--force` is used
     if (options.runUpdateOnFailed || options.force) {
-      const { success, metrics } = await runIntegrationTests({
+      const { success, metrics, removedEnvironments } = await runIntegrationTests({
         pool,
         tests: testsToRun,
         regions: options.testRegions,
@@ -197,6 +197,9 @@ async function run(options: ReturnType<typeof parseCliArgs>) {
       });
       testsSucceeded = success;
 
+      // Print summary of removed environments due to bootstrap errors
+      printRemovedEnvironmentsSummary(removedEnvironments);
+
       if (options.clean === false) {
         logger.warning('Not cleaning up stacks since "--no-clean" was used');
       }

packages/@aws-cdk/integ-runner/lib/cli.ts

@@ -0,0 +1,33 @@
+import { ToolkitError } from '@aws-cdk/toolkit-lib';
+
+/**
+ * Result of bootstrap error detection
+ */
+export interface BootstrapErrorInfo {
+  readonly isBootstrapError: boolean;
+  readonly region?: string;
+  readonly account?: string;
+  readonly message: string;
+}
+
+/**
+ * Detects if an error is a bootstrap-related error
+ */
+export function detectBootstrapError(error: unknown): BootstrapErrorInfo {
+  // Check for strongly-typed BootstrapError
+  if (ToolkitError.isBootstrapError(error)) {
+    return {
+      isBootstrapError: true,
+      region: error.environment.region,
+      account: error.environment.account,
+      message: error.message,
+    };
+  }
+
+  const errorMessage = error instanceof Error ? error.message : String(error);
+
+  return {
+    isBootstrapError: false,
+    message: errorMessage,
+  };
+}

packages/@aws-cdk/integ-runner/lib/workers/bootstrap-error-detection.ts

@@ -2,6 +2,7 @@ import { format } from 'util';
 import type { ResourceImpact } from '@aws-cdk/cloudformation-diff';
 import * as chalk from 'chalk';
 import * as logger from '../logger';
+import type { TestEnvironment, RemovedEnvironmentInfo } from './environment-pool';
 import type { IntegTestInfo } from '../runner/integration-tests';
 
 /**
@@ -37,6 +38,41 @@ export interface IntegTestWorkerConfig extends IntegTestInfo {
   readonly destructiveChanges?: DestructiveChange[];
 }
 
+/**
+ * Request to remove an environment from the pool
+ */
+export interface EnvironmentRemovalRequest {
+  /**
+   * The environment to remove
+   */
+  readonly environment: TestEnvironment;
+
+  /**
+   * Human-readable reason for removal
+   */
+  readonly reason: string;
+
+  /**
+   * AWS account ID if known
+   */
+  readonly account?: string;
+}
+
+/**
+ * A test failure that can potentially be retried
+ */
+export interface RetryableTestFailure extends IntegTestInfo {
+  /**
+   * The environment where the failure occurred
+   */
+  readonly failedEnvironment: TestEnvironment;
+
+  /**
+   * Human-readable error message
+   */
+  readonly errorMessage: string;
+}
+
 /**
  * Information on any destructive changes
  */
@@ -118,6 +154,16 @@ export interface IntegBatchResponse {
    * list represents metrics from a single worker (account + region).
    */
   readonly metrics: IntegRunnerMetrics[];
+
+  /**
+   * Tests that failed but may succeed if retried in a different environment
+   */
+  readonly retryableFailures?: RetryableTestFailure[];
+
+  /**
+   * Environments that should be removed from the pool
+   */
+  readonly environmentRemovals?: EnvironmentRemovalRequest[];
 }
 
 /**
@@ -344,3 +390,24 @@ export function formatError(error: any): string {
 
   return `${name}: ${message}`;
 }
+
+/**
+ * Prints a summary of environments that were removed due to bootstrap errors
+ */
+export function printRemovedEnvironmentsSummary(removedEnvironments: RemovedEnvironmentInfo[]): void {
+  if (removedEnvironments.length === 0) {
+    return;
+  }
+
+  logger.warning('\n%s', chalk.bold('Environments removed due to bootstrap errors:'));
+
+  for (const env of removedEnvironments) {
+    const profileStr = env.profile ? `${env.profile}/` : '';
+    const accountStr = env.account ? `aws://${env.account}/${env.region}` : env.region;
+
+    logger.warning('  • %s%s', profileStr, env.region);
+    logger.warning('    Run: %s', chalk.blue(`cdk bootstrap ${accountStr}`));
+  }
+
+  logger.warning('');
+}

packages/@aws-cdk/integ-runner/lib/workers/common.ts

@@ -0,0 +1,93 @@
+/**
+ * Identifies a specific profile+region combination (an "environment" for test execution)
+ */
+export interface TestEnvironment {
+  readonly profile?: string;
+  readonly region: string;
+}
+
+/**
+ * Information about why an environment was removed
+ */
+export interface RemovedEnvironmentInfo extends TestEnvironment {
+  readonly reason: string;
+  readonly account?: string;
+  readonly removedAt: Date;
+}
+
+/**
+ * Manages a pool of test environments for integration test workers.
+ *
+ * This class serves as a centralized pool for test environments, handling:
+ * - Tracking which environments are available vs removed
+ * - Recording removal reasons for reporting
+ *
+ * Future extensions could include:
+ * - Load balancing across environments
+ * - Rate limiting per environment
+ * - Environment health scoring
+ * - Automatic environment recovery
+ */
+export class EnvironmentPool {
+  private readonly availableEnvironments: Set<string>;
+  private readonly removedEnvironments: Map<string, RemovedEnvironmentInfo> = new Map();
+
+  constructor(environments: TestEnvironment[]) {
+    this.availableEnvironments = new Set(environments.map(e => this.makeKey(e)));
+  }
+
+  /**
+   * Creates a unique key for a profile+region combination
+   */
+  private makeKey(env: TestEnvironment): string {
+    return `${env.profile ?? 'default'}:${env.region}`;
+  }
+
+  /**
+   * Parses a key back into a TestEnvironment
+   */
+  private parseKey(key: string): TestEnvironment {
+    const [profile, region] = key.split(':');
+    return {
+      profile: profile === 'default' ? undefined : profile,
+      region,
+    };
+  }
+
+  /**
+   * Marks an environment as removed (unavailable for future tests)
+   */
+  public removeEnvironment(env: TestEnvironment, reason: string, account?: string): void {
+    const key = this.makeKey(env);
+    if (this.availableEnvironments.has(key)) {
+      this.availableEnvironments.delete(key);
+      this.removedEnvironments.set(key, {
+        ...env,
+        reason,
+        account,
+        removedAt: new Date(),
+      });
+    }
+  }
+
+  /**
+   * Checks if an environment is still available
+   */
+  public isAvailable(env: TestEnvironment): boolean {
+    return this.availableEnvironments.has(this.makeKey(env));
+  }
+
+  /**
+   * Gets all available environments
+   */
+  public getAvailableEnvironments(): TestEnvironment[] {
+    return Array.from(this.availableEnvironments).map(key => this.parseKey(key));
+  }
+
+  /**
+   * Gets all removed environments with their removal info
+   */
+  public getRemovedEnvironments(): RemovedEnvironmentInfo[] {
+    return Array.from(this.removedEnvironments.values());
+  }
+}