Added support for user defined environments in CI/CD pipeline

Author: jmgtan

.projen/tasks.json

@@ -206,6 +206,36 @@ You need to also provide the accounts information in the cdk.json in the form of
 }
 ```
 
+## User Defined Stages
+
+To define multiple stages (which can also be deployed in different AWS accounts by following the bootstrap command in the previous section), configure the `cdk.json` file with the following:
+
+```json
+{
+  "environments": [
+    {
+      "stageName": "<STAGE_NAME_1>",
+      "account": "<STAGE_ACCOUNT_ID>",
+      "region": "<REGION>",
+      "triggerIntegTest": "<OPTIONAL_BOOLEAN_CAN_BE_OMMITTED>"
+    },
+    {
+      "stageName": "<STAGE_NAME_2>",
+      "account": "<STAGE_ACCOUNT_ID>",
+      "region": "<REGION>",
+      "triggerIntegTest": "<OPTIONAL_BOOLEAN_CAN_BE_OMMITTED>"
+    },
+    {
+      "stageName": "<STAGE_NAME_3>",
+      "account": "<STAGE_ACCOUNT_ID>",
+      "region": "<REGION>",
+      "triggerIntegTest": "<OPTIONAL_BOOLEAN_CAN_BE_OMMITTED>"
+    }
+
+  ]
+}
+```
+
 ## Defining a CDK Stack for the Spark application
 
 The `SparkCICDPipeline` construct deploys an application stack, which contains your business logic, into staging and production environments.

LICENSE

@@ -5,7 +5,7 @@ import { RemovalPolicy, ResourceEnvironment } from 'aws-cdk-lib';
 import { Key } from 'aws-cdk-lib/aws-kms';
 import { ILogGroup, LogGroup } from 'aws-cdk-lib/aws-logs';
 import { Bucket, BucketEncryption, IBucket } from 'aws-cdk-lib/aws-s3';
-import { CodeBuildStep, CodePipeline } from 'aws-cdk-lib/pipelines';
+import { CodeBuildStep, CodePipeline, StageDeployment } from 'aws-cdk-lib/pipelines';
 import { Construct } from 'constructs';
 import { SparkEmrCICDPipelineProps } from './spark-emr-cicd-pipeline-props';
 import { AccessLogsBucket } from '../../../storage';
@@ -18,6 +18,15 @@ import {
 } from '../../../utils';
 import { DEFAULT_SPARK_IMAGE, SparkImage } from '../emr-releases';
 
+/**
+ * User defined CI/CD environment stages
+ */
+interface CICDEnvironment {
+  stageName: string;
+  account: string;
+  region: string;
+  triggerIntegTest?: boolean;
+}
 
 /**
  * A CICD Pipeline to test and deploy a Spark application on Amazon EMR in cross-account environments using CDK Pipelines.
@@ -211,36 +220,89 @@ export class SparkEmrCICDPipeline extends TrackedConstruct {
       },
     });
 
-    // Create the Staging stage of the CICD
-    const staging = new ApplicationStage(this, 'Staging', {
-      env: this.getAccountFromContext('staging'),
+    try {
+      const environments = this.getUserDefinedEnvironmentsFromContext();
+
+      for (const e of environments) {
+        this.attachStageToPipeline(e.stageName.toUpperCase(), {
+          account: e.account,
+          region: e.region,
+        }, e.triggerIntegTest || false, buildStage, props);
+      }
+    } catch (e) {
+      [,, this.integrationTestStage] = this.attachStageToPipeline('Staging', this.getAccountFromContext('staging'), true, buildStage, props);
+      this.attachStageToPipeline('Prod', this.getAccountFromContext('prod'), false, buildStage, props);
+    }
+
+    // // Create the Staging stage of the CICD
+    // const staging = new ApplicationStage(this, 'Staging', {
+    //   env: this.getAccountFromContext('staging'),
+    //   applicationStackFactory: props.applicationStackFactory,
+    //   outputsEnv: props.integTestEnv,
+    //   stage: CICDStage.STAGING,
+    // });
+    // const stagingDeployment = this.pipeline.addStage(staging);
+
+    // if (props.integTestScript) {
+    //   // Extract the path and script name from the integration tests script path
+    //   const [integPath, integScript] = SparkEmrCICDPipeline.extractPath(props.integTestScript);
+
+    //   this.integrationTestStage = new CodeBuildStep('IntegrationTests', {
+    //     input: buildStage.addOutputDirectory(integPath),
+    //     commands: [`chmod +x ${integScript} && ./${integScript}`],
+    //     envFromCfnOutputs: staging.stackOutputsEnv,
+    //     rolePolicyStatements: props.integTestPermissions,
+    //   });
+    //   // Add a post step to run the integration tests
+    //   stagingDeployment.addPost(this.integrationTestStage);
+    // }
+
+    // // Create the Production stage of the CICD
+    // this.pipeline.addStage(new ApplicationStage(this, 'Production', {
+    //   env: this.getAccountFromContext('prod'),
+    //   applicationStackFactory: props.applicationStackFactory,
+    //   stage: CICDStage.PROD,
+    // }));
+
+  }
+
+  /**
+   * Attaches the given stage to the pipeline
+   * @param stageName
+   * @param resourceEnvironment
+   * @param attachIntegTest
+   * @param buildStage
+   * @param props
+   * @returns {Array} contains array of ApplicationStage, StageDeployment, and optional integration test step
+   */
+  private attachStageToPipeline(stageName: string, resourceEnvironment: ResourceEnvironment
+    , attachIntegTest: boolean, buildStage: CodeBuildStep
+    , props: SparkEmrCICDPipelineProps): [ApplicationStage, StageDeployment, CodeBuildStep|undefined] {
+    const applicationStage = new ApplicationStage(this, stageName, {
+      env: resourceEnvironment,
       applicationStackFactory: props.applicationStackFactory,
-      outputsEnv: props.integTestEnv,
-      stage: CICDStage.STAGING,
+      outputsEnv: (attachIntegTest && props.integTestScript) ? props.integTestEnv : undefined,
+      stage: CICDStage.of(stageName.toUpperCase()),
     });
-    const stagingDeployment = this.pipeline.addStage(staging);
+    const stageDeployment = this.pipeline.addStage(applicationStage);
 
-    if (props.integTestScript) {
+    let integrationTestStage:CodeBuildStep|undefined = undefined;
+
+    if (attachIntegTest && props.integTestScript) {
       // Extract the path and script name from the integration tests script path
       const [integPath, integScript] = SparkEmrCICDPipeline.extractPath(props.integTestScript);
 
-      this.integrationTestStage = new CodeBuildStep('IntegrationTests', {
+      integrationTestStage = new CodeBuildStep(`${stageName}-IntegrationTests`, {
         input: buildStage.addOutputDirectory(integPath),
         commands: [`chmod +x ${integScript} && ./${integScript}`],
-        envFromCfnOutputs: staging.stackOutputsEnv,
+        envFromCfnOutputs: applicationStage.stackOutputsEnv,
         rolePolicyStatements: props.integTestPermissions,
       });
       // Add a post step to run the integration tests
-      stagingDeployment.addPost(this.integrationTestStage);
+      stageDeployment.addPost(integrationTestStage);
     }
 
-    // Create the Production stage of the CICD
-    this.pipeline.addStage(new ApplicationStage(this, 'Production', {
-      env: this.getAccountFromContext('prod'),
-      applicationStackFactory: props.applicationStackFactory,
-      stage: CICDStage.PROD,
-    }));
-
+    return [applicationStage, stageDeployment, integrationTestStage];
   }
 
   /**
@@ -251,4 +313,14 @@ export class SparkEmrCICDPipeline extends TrackedConstruct {
     if (!account) throw new Error(`Missing context variable ${name}`);
     return account;
   }
+
+  private getUserDefinedEnvironmentsFromContext(): CICDEnvironment[] {
+    const environments = this.node.tryGetContext('environments') as CICDEnvironment[];
+
+    if (!environments) {
+      throw new Error('Missing context variable environments');
+    }
+
+    return environments;
+  }
 }

framework/API.md

@@ -10,9 +10,31 @@ import { ApplicationStackFactory } from './application-stack-factory';
 /**
  * The list of CICD Stages used in CICD Pipelines.
  */
-export enum CICDStage {
-  STAGING = 'staging',
-  PROD = 'prod',
+export class CICDStage {
+
+  /**
+   * Prod stage
+   */
+  public static readonly PROD = CICDStage.of('PROD');
+
+  /**
+   * Staging stage
+   */
+  public static readonly STAGING = CICDStage.of('STAGING');
+
+  /**
+   * Custom stage
+   * @param stage the stage inside the pipeline
+   * @returns
+   */
+  public static of(stage: string) {
+    return new CICDStage(stage);
+  }
+
+  /**
+   * @param stage the stage inside the pipeline
+   */
+  private constructor(public readonly stage: string) {}
 }
 
 /**

framework/LICENSE

@@ -102,7 +102,7 @@ NagSuppressions.addResourceSuppressionsByPath(
 
 NagSuppressions.addResourceSuppressionsByPath(
   stack,
-  '/Stack/TestConstruct/CodePipeline/Pipeline/Staging/IntegrationTests/IntegrationTests/Role',
+  '/Stack/TestConstruct/CodePipeline/Pipeline/Staging/Staging-IntegrationTests/Staging-IntegrationTests/Role',
   [{ id: 'AwsSolutions-IAM5', reason: 'This role is provided by CDK Pipeline construt' }],
   true,
 );