fix: cicd pipeline and quicksight custom resources (#597)

* fix cicd pipeline and quicksight custom resources

Author: vgkowski

refarch/README.md

@@ -78,7 +78,7 @@ To disable, for example, the data visualization module, the following argument h
    **NOTE:
    Step 4 and 5 only have to be executed if the data visualization module has been installed.**
 
-1. Configure Amazon QuickSight:
+1. <a name="quicksight">Configure Amazon QuickSight:</a>
     * To link the clean data S3 bucket to the QuickSight account:
         * Visit the [QuickSight web console](https://quicksight.aws.amazon.com)
         * Click on your username in the upper right corner
@@ -164,10 +164,20 @@ The pipeline and the reference architecture can live in different accounts or in
    * **<DEV_ACCOUNT>** and **<PROD_ACCOUNT>**: with the account that will be deployed.
    * Note that you can have only a single account to be both CI/CD and DEV. If you don't want PROD account, you can comment out the line `deploy_envs.append(prod_env)` in the file `refarch/aws-native/app.py`
 
-1. Run `cdk deploy --profile <CICD_ACCOUNT_PROFILE_NAME> araPipelineStack` using the credentials for your CICD account.
+1. Push the code into the repository used by the CICD pipeline.
+
+```
+git push RepositoryName RepositoryBranch
+```
+
+2. Run `cdk deploy --profile <CICD_ACCOUNT_PROFILE_NAME> araPipelineStack` using the credentials for your CICD account.
       * This will deploy the stack containing the pipeline to your CICD account
       * After the pipeline has been successfully deploy, it will fetch code from the specified repo and deploy to the target account.
 
+2. Configure Quicksight as mentioned in [step 4](#quicksight) when provisioning the application directly
+
+
+
 #### Adding users to Kibana
 
 The main CDK stack also deploys the streaming module (if not explicitly disabled), which includes:

refarch/aws-native/app.py

@@ -43,8 +43,9 @@ def make_env(scope: Construct, context_key: str):
     },
 )
 
+deploy_envs = []
+
 if app.node.try_get_context('EnableCICD') == 'true':
-    deploy_envs = []
 
     cicd_account_context = app.node.try_get_context('CICD')
     if cicd_account_context is None:
@@ -54,16 +55,16 @@ def make_env(scope: Construct, context_key: str):
     deploy_envs.append(dev_env)
 
     # Comment out to deploy only to dev environment
-    prod_env = make_env(app, 'PROD')
-    deploy_envs.append(prod_env)
+    # prod_env = make_env(app, 'PROD')
+    # deploy_envs.append(prod_env)
+
+PipelineStack(app, "araPipelineStack",
+            env={
+                'account': cicd_account_context.get('account'),
+                'region': cicd_account_context.get('region')
+            },
+            deploy_envs=deploy_envs)
 
-    PipelineStack(app, "araPipelineStack",
-                  env={
-                      'account': cicd_account_context.get('account'),
-                      'region': cicd_account_context.get('region')
-                  },
-                  deploy_envs=deploy_envs)
-else:
-    DataLake(app, "ara")
+DataLake(app, "ara")
 
 app.synth()

refarch/aws-native/cdk.context.json

@@ -6,22 +6,21 @@
   },
   "DEV": {
     "name": "dev",
-    "region": "eu-west-1",
+    "region": "<DEV_REGION>",
     "account": "<DEV_ACCOUNT>"
   },
   "PROD": {
     "name": "prod",
-    "region": "eu-west-1",
+    "region": "<PROD_REGION>",
     "account": "<PROD_ACCOUNT>"
   },
-  "availability-zones:account=<DEV_ACCOUNT>:region=eu-west-1": [
-    "eu-west-1a",
-    "eu-west-1b",
-    "eu-west-1c"
+  "availability-zones:account=<DEV_ACCOUNT>:region=<DEV_REGION>": [
+    "<DEV_REGION>a",
+    "<DEV_REGION>b",
+    "<DEV_REGION>c"
   ],
-  "availability-zones:account=<PROD_ACCOUNT>:region=eu-west-1": [
-    "eu-west-1a",
-    "eu-west-1b",
-    "eu-west-1c"
-  ]
+  "availability-zones:account=<PROD_ACCOUNT>:region=<PROD_REGION>": [
+    "<PROD_REGION>a",
+    "<PROD_REGION>b",
+    "<PROD_REGION>c"
 }

refarch/aws-native/cdk.json

@@ -16,4 +16,4 @@
     "QuickSightIdentityRegion": "<QUICKSIGHT_IDENTITY_REGION>",
     "EnableDeploymentTracking": "true"
   }
-}
+}

refarch/aws-native/cicd/pipeline.py

@@ -45,14 +45,14 @@ def __init__(self, scope: Construct, id: str, deploy_envs: list, **kwargs) -> No
                                                     'cd refarch/aws-native',
                                                     'pip install -r requirements.txt',
                                                     'which npx',
-                                                    'npm install -g aws-cdk',
+                                                    'npm install -g aws-cdk@2.51.0',
                                                     'cdk synth'
                                                 ],
                                                 primary_output_directory='refarch/aws-native/cdk.out'),
                                 cross_account_keys=True,
-
+                                self_mutation=True
                                 )
-
+    
         for env in deploy_envs:
             pipeline.add_stage(stage=PipelineStage(self, 'AnalyticsPipelineStage', env=Environment(
                 account=env.account,

refarch/aws-native/common/common_cdk/data_lake.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: MIT-0
 
 from aws_cdk import aws_dynamodb as _dynamodb
+from aws_cdk import aws_iam as _iam
 from aws_cdk import CfnOutput, Stack, Tags
 from constructs import Construct
 
@@ -101,6 +102,10 @@ def __init__(self, scope: Construct, id: str, **kwargs) -> None:
                                           quicksight_username=quicksight_username,
                                           quicksight_identity_region=quicksight_identity_region)
 
+            quicksight_role = _iam.Role.from_role_name(self,'QuicksightRole', role_name='aws-quicksight-service-role-v0')
+            quicksight_role.node.add_dependency(dataviz_stack)
+            data_lake.clean_s3_bucket.encryption_key.grant_decrypt(quicksight_role)
+
             CfnOutput(self, 'QuickSight-Security-Group-Id',
                       value=dataviz_stack.quicksight_security_group_id)
 

refarch/aws-native/dataviz/dataviz_cdk/qs_athena_dataset.py

@@ -32,17 +32,14 @@ def __init__(
         super().__init__(scope, id, **kwargs)
 
         aws_account_id = Aws.ACCOUNT_ID
-        uniquestring = datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S')
-        athena_dataset_id = athena_dataset_name + uniquestring
-        athena_dataset_physical_id = athena_dataset_name + uniquestring
 
         quicksight_athena_dataset = cr.AwsCustomResource(self, 'AthenaDataSet',
                                                    on_create={
                                                        "service": "QuickSight",
                                                        "action": "createDataSet",
                                                        "parameters": {
                                                            "AwsAccountId": aws_account_id,
-                                                           "DataSetId": athena_dataset_id,
+                                                           "DataSetId": athena_dataset_name,
                                                            "Name": athena_dataset_name,
                                                            "ImportMode": "DIRECT_QUERY",
                                                            "PhysicalTableMap": {
@@ -73,16 +70,16 @@ def __init__(
 
                                                        },
                                                        "physical_resource_id": cr.PhysicalResourceId.of(
-                                                           athena_dataset_physical_id)},
+                                                           athena_dataset_name)},
                                                    on_delete={
                                                        "service": "QuickSight",
                                                        "action": "deleteDataSet",
                                                        "parameters": {
                                                            "AwsAccountId": aws_account_id,
-                                                           "DataSetId": athena_dataset_id
+                                                           "DataSetId": athena_dataset_name
                                                        },
                                                        "physical_resource_id": cr.PhysicalResourceId.of(
-                                                           athena_dataset_physical_id)},
+                                                           athena_dataset_name)},
                                                    policy=iam_policy
                                                    )