Additional lake formation permission support.

paulu-aws · paulu-aws · commit 98aa9563194b · 2020-05-14T10:51:54.000-05:00
diff --git a/DeployChemblOpenTargetsEnv.sh b/DeployChemblOpenTargetsEnv.sh
@@ -1,6 +1,9 @@
+#!/bin/sh
 npm run build
 cdk bootstrap
 cdk deploy BaselineStack --require-approval never
+currentPrincipalArn=$(aws sts get-caller-identity --query Arn --output text)
+jq '.context.starterLakeFormationAdmin = $currentPrincipalArn' --arg currentPrincipalArn $currentPrincipalArn cdk.json > tmp.$$.json && mv tmp.$$.json cdk.json
 cdk deploy CoreDataLake --require-approval never
 cdk deploy ChemblStack --require-approval never
 cdk deploy OpenTargetsStack --require-approval never
diff --git a/bin/aws.ts b/bin/aws.ts
@@ -15,7 +15,7 @@ const baseline = new BaselineStack(app, 'BaselineStack');
 
 
 const coreDataLake = new DataLakeStack(app, 'CoreDataLake', {
-
+    starterLakeFormationAdminPrincipalArn: app.node.tryGetContext("starterLakeFormationAdmin")
 });
 
 const chemblStack = new ChemblStack(app, 'ChemblStack', {
@@ -36,6 +36,9 @@ const analyticsStack = new AnalyticsStack(app, 'AnalyticsStack', {
 });
 
 
+
+
+
 chemblStack.grantIamRead(analyticsStack.NotebookRole);
 openTargetsStack.grantIamRead(analyticsStack.NotebookRole);
 
@@ -46,44 +49,44 @@ openTargetsStack.grantIamRead(analyticsStack.NotebookRole);
 
 
 
-const exampleUser = iam.User.fromUserName(coreDataLake, 'exampleGrantee', 'paul1' );
+// const exampleUser = iam.User.fromUserName(coreDataLake, 'exampleGrantee', 'paul1' );
 
-var exampleGrant: DataLakeEnrollment.TablePermissionGrant = {
-    tables: ["association_data", "evidence_data","target_list","disease_list"],
-    DatabasePermissions: [DataLakeEnrollment.DatabasePermission.Alter, DataLakeEnrollment.DatabasePermission.CreateTable, DataLakeEnrollment.DatabasePermission.Drop],
-    GrantableDatabasePermissions: [DataLakeEnrollment.DatabasePermission.Alter, DataLakeEnrollment.DatabasePermission.CreateTable, DataLakeEnrollment.DatabasePermission.Drop],
-    TablePermissions: [DataLakeEnrollment.TablePermission.Select, DataLakeEnrollment.TablePermission.Insert, DataLakeEnrollment.TablePermission.Delete],
-    GrantableTablePermissions: [DataLakeEnrollment.TablePermission.Select]
-};
+// var exampleGrant: DataLakeEnrollment.TablePermissionGrant = {
+//     tables: ["association_data", "evidence_data","target_list","disease_list"],
+//     DatabasePermissions: [DataLakeEnrollment.DatabasePermission.Alter, DataLakeEnrollment.DatabasePermission.CreateTable, DataLakeEnrollment.DatabasePermission.Drop],
+//     GrantableDatabasePermissions: [DataLakeEnrollment.DatabasePermission.Alter, DataLakeEnrollment.DatabasePermission.CreateTable, DataLakeEnrollment.DatabasePermission.Drop],
+//     TablePermissions: [DataLakeEnrollment.TablePermission.Select, DataLakeEnrollment.TablePermission.Insert, DataLakeEnrollment.TablePermission.Delete],
+//     GrantableTablePermissions: [DataLakeEnrollment.TablePermission.Select]
+// };
 
-openTargetsStack.grantTablePermissions(exampleUser, exampleGrant);
+// openTargetsStack.grantTablePermissions(exampleUser, exampleGrant);
 
 
 
 
-// In the example below, we are using the compound_structures table from ChEMBL. It has the following table definition:
-// ['molregno', 'molfile', 'standard_inchi', 'standard_inchi_key', 'canonical_smiles']
-// Lets say we want to give a principal ONLY select permissions to everything in the compound_structures table BUT the 'canonical_smiles' column.
+// // In the example below, we are using the compound_structures table from ChEMBL. It has the following table definition:
+// // ['molregno', 'molfile', 'standard_inchi', 'standard_inchi_key', 'canonical_smiles']
+// // Lets say we want to give a principal ONLY select permissions to everything in the compound_structures table BUT the 'canonical_smiles' column.
 
-var exampleTableWithColumnsGrant: DataLakeEnrollment.TableWithColumnPermissionGrant = {
-    table: "chembl_25_public_compound_structures",
-    // Note that we are NOT including 'canonical_smiles'. That effectivley prevents this user from querying that column.
-    columns: ['molregno', 'molfile', 'standard_inchi', 'standard_inchi_key'],
-    DatabasePermissions: [],
-    GrantableDatabasePermissions: [],
-    TableColumnPermissions: [DataLakeEnrollment.TablePermission.Select],
-    GrantableTableColumnPermissions: []
-};
+// var exampleTableWithColumnsGrant: DataLakeEnrollment.TableWithColumnPermissionGrant = {
+//     table: "chembl_25_public_compound_structures",
+//     // Note that we are NOT including 'canonical_smiles'. That effectivley prevents this user from querying that column.
+//     columns: ['molregno', 'molfile', 'standard_inchi', 'standard_inchi_key'],
+//     DatabasePermissions: [],
+//     GrantableDatabasePermissions: [],
+//     TableColumnPermissions: [DataLakeEnrollment.TablePermission.Select],
+//     GrantableTableColumnPermissions: []
+// };
 
-var exampleTableWithColumnsGrant_WithWildCard: DataLakeEnrollment.TableWithColumnPermissionGrant = {
-    table: "chembl_25_public_compound_structures",
-    wildCardFilter: DataLakeEnrollment.TableWithColumnFilter.Exclude,
-    columns: ['canonical_smiles'],
-    DatabasePermissions: [],
-    GrantableDatabasePermissions: [],
-    TableColumnPermissions: [DataLakeEnrollment.TablePermission.Select],
-    GrantableTableColumnPermissions: []
-};
+// var exampleTableWithColumnsGrant_WithWildCard: DataLakeEnrollment.TableWithColumnPermissionGrant = {
+//     table: "chembl_25_public_compound_structures",
+//     wildCardFilter: DataLakeEnrollment.TableWithColumnFilter.Exclude,
+//     columns: ['canonical_smiles'],
+//     DatabasePermissions: [],
+//     GrantableDatabasePermissions: [],
+//     TableColumnPermissions: [DataLakeEnrollment.TablePermission.Select],
+//     GrantableTableColumnPermissions: []
+// };
 
-// Note that exampleTableWithColumnsGrant exampleTableWithColumnsGrant_WithWildCard grants the same effecitve permissions. One just uses a the wildcard.
-chemblStack.grantTableWithColumnPermissions(exampleUser, exampleTableWithColumnsGrant);
+// // Note that exampleTableWithColumnsGrant exampleTableWithColumnsGrant_WithWildCard grants the same effecitve permissions. One just uses a the wildcard.
+// chemblStack.grantTableWithColumnPermissions(exampleUser, exampleTableWithColumnsGrant);
diff --git a/cdk.json b/cdk.json
@@ -1,3 +1,6 @@
 {
-  "app": "npx ts-node bin/aws.ts"
+  "app": "npx ts-node bin/aws.ts",
+  "context": {
+    "starterLakeFormationAdmin": "XXXX-this value gets populated by DeployChemblOpenTargetsEnv.sh script-XXXXX"
+  }
 }
diff --git a/lib/chembl-25-stack.ts b/lib/chembl-25-stack.ts
@@ -42,7 +42,9 @@ export class ChemblStack extends DataSetStack{
 				"--DL_REGION": cdk.Stack.of(this).region,
 				"--GLUE_SRC_DATABASE": "chembl_25_src"
 			}	    	
-	    });
+		});
+		
+		
 	}
 }
 
diff --git a/lib/constructs/data-lake-enrollment.ts b/lib/constructs/data-lake-enrollment.ts
@@ -27,6 +27,7 @@ export class DataLakeEnrollment extends cdk.Construct {
         this.CoarseIamPolciesApplied = false;
 
     }
+ 
 
     public createCoarseIamPolicy(){
 
@@ -277,7 +278,7 @@ export class DataLakeEnrollment extends cdk.Construct {
 
     }
 
-    public grantDatabasePermission(principal: iam.IPrincipal, permissionGrant: DataLakeEnrollment.DatabasePermissionGrant){
+    public grantDatabasePermission(principal: iam.IPrincipal, permissionGrant: DataLakeEnrollment.DatabasePermissionGrant, includeSourceDb: boolean = false){
 
 
         var grantIdPrefix = ""
@@ -293,7 +294,12 @@ export class DataLakeEnrollment extends cdk.Construct {
 
         if(resolvedPrincipalType === iam.Role) {
             const resolvedPrincipal = principal as  iam.Role;
-            grantIdPrefix = `${resolvedPrincipal.roleArn}-${this.DataSetName}`
+
+            if(permissionGrant.GrantResourcePrefix){
+                grantIdPrefix = `${permissionGrant.GrantResourcePrefix}-${this.DataSetName}`
+            }else{
+                grantIdPrefix = `${resolvedPrincipal.roleName}-${this.DataSetName}`
+            }            
             dataLakePrincipal = { dataLakePrincipalIdentifier: resolvedPrincipal.roleArn };
 		}
 
@@ -305,6 +311,18 @@ export class DataLakeEnrollment extends cdk.Construct {
 
         this.createLakeFormationPermission(`${grantIdPrefix}-databaseGrant`,dataLakePrincipal , databaseResourceProperty, permissionGrant.DatabasePermissions, permissionGrant.GrantableDatabasePermissions)
 
+        if(includeSourceDb){
+
+            databaseResourceProperty = {
+                //dataLocationResource: {resourceArn: this.DataEnrollment.DataLakeBucketName},
+                databaseResource: {name: this.DataEnrollment.Dataset_Source.databaseName}
+            };
+
+            this.createLakeFormationPermission(`${grantIdPrefix}-databaseSrcGrant`,dataLakePrincipal , databaseResourceProperty, permissionGrant.DatabasePermissions, permissionGrant.GrantableDatabasePermissions)
+
+        }
+
+
     }
 
 
@@ -462,6 +480,7 @@ export namespace DataLakeEnrollment
     export interface DatabasePermissionGrant {
         DatabasePermissions: Array<DatabasePermission>;
         GrantableDatabasePermissions: Array<DatabasePermission>;
+        GrantResourcePrefix?: string;
     }
 
 
diff --git a/lib/constructs/rds-data-set-enrollment.ts b/lib/constructs/rds-data-set-enrollment.ts
@@ -34,7 +34,7 @@ export class RDSPostgresDataSetEnrollment extends DataLakeEnrollment {
 			});
             
         }
-        
+		
         this.DataEnrollment = new DataSetEnrollment(this, 'rdsDatasetEnrollment', {
 			dataLakeBucket: props.dataLakeBucket,
 			dataSetName: dataSetName,
@@ -61,7 +61,15 @@ export class RDSPostgresDataSetEnrollment extends DataLakeEnrollment {
 			GlueScriptArguments: props.GlueScriptArguments
 			
 		});        
-        
+						
+
 		this.createCoarseIamPolicy();
+
+        this.grantDatabasePermission(this.DataEnrollment.DataSetGlueRole,  {		     
+			DatabasePermissions: [DataLakeEnrollment.DatabasePermission.All],
+			GrantableDatabasePermissions: [DataLakeEnrollment.DatabasePermission.All],
+			GrantResourcePrefix: `${props.DataSetName}RoleGrant`
+	   }, true);
+
 	}
 }
diff --git a/lib/constructs/s3-data-set-enrollment.ts b/lib/constructs/s3-data-set-enrollment.ts
@@ -47,7 +47,7 @@ export class S3dataSetEnrollment extends DataLakeEnrollment{
             
         }
 		
-		this.DataEnrollment = new DataSetEnrollment(this, 'openTargetsEnrollment', {
+		this.DataEnrollment = new DataSetEnrollment(this, `${props.DataSetName}-s3Enrollment`, {
 		    dataLakeBucket: props.dataLakeBucket,
 			dataSetName: dataSetName,
 			SourceAccessPolicy: s3AccessPolicy,
@@ -58,6 +58,15 @@ export class S3dataSetEnrollment extends DataLakeEnrollment{
 			GlueScriptArguments: props.GlueScriptArguments
 		});
 	
-	    this.createCoarseIamPolicy();
+        this.createCoarseIamPolicy();
+        
+
+        this.grantDatabasePermission(this.DataEnrollment.DataSetGlueRole,  {		     
+		     DatabasePermissions: [DataLakeEnrollment.DatabasePermission.All],
+             GrantableDatabasePermissions: [DataLakeEnrollment.DatabasePermission.All],
+             GrantResourcePrefix: `${props.DataSetName}RoleGrant`
+		}, true);
+
+
 	}
 }
diff --git a/lib/stacks/datalake-stack.ts b/lib/stacks/datalake-stack.ts
@@ -8,13 +8,15 @@ import cfn = require("@aws-cdk/aws-cloudformation");
 import s3 = require('@aws-cdk/aws-s3');
 import s3assets = require('@aws-cdk/aws-s3-assets');
 import lakeformation = require('@aws-cdk/aws-lakeformation');
-import lambda = require('@aws-cdk/aws-lambda')
+import { CustomResource } from '@aws-cdk/core';
+import * as cr from '@aws-cdk/custom-resources';
+import lambda = require('@aws-cdk/aws-lambda');
 import fs = require('fs');
 
 import { DataSetEnrollmentProps, DataSetEnrollment } from '../constructs/data-set-enrollment';
 
 export interface DataLakeStackProps extends cdk.StackProps {
-
+  starterLakeFormationAdminPrincipalArn: string;
 }
 
 export class DataLakeStack extends cdk.Stack {
@@ -25,6 +27,7 @@ export class DataLakeStack extends cdk.Stack {
   public readonly LakeFormationResource: lakeformation.CfnResource;
   public readonly PrimaryAthenaWorkgroup: athena.CfnWorkGroup;
   private readonly bucketRole: iam.Role; 
+  private readonly starterAdminArn: string;
   
   public grantAthenaResultsBucketPermission(principal: iam.IPrincipal){
     
@@ -63,9 +66,14 @@ export class DataLakeStack extends cdk.Stack {
     super(scope, id, props);
 
     this.DataLakeBucket = new s3.Bucket(this, 'dataLakeBucket');
-    this.AthenaResultsBucket = new s3.Bucket(this, 'athenaResultsBucket');
-    
-    
+    this.AthenaResultsBucket = new s3.Bucket(this, 'athenaResultsBucket');        
+   
+    new lakeformation.CfnDataLakeSettings(this, 'starterAdminPermission', {
+      admins: [{
+        dataLakePrincipalIdentifier: props.starterLakeFormationAdminPrincipalArn        
+      }]
+    });
+
     const coarseAthenaResultBucketAccess = {
             "Version": "2012-10-17",
             "Statement": [
diff --git a/lib/stacks/dataset-stack.ts b/lib/stacks/dataset-stack.ts
@@ -24,7 +24,7 @@ export class DataSetStack extends cdk.Stack {
     this.DataLake = props.DataLake;
   }
 
-
+  
   public grantDatabasePermissions( principal: iam.IPrincipal, permissionGrant: DataLakeEnrollment.DatabasePermissionGrant){
     this.Enrollment.grantDatabasePermission(principal, permissionGrant);
   }
diff --git a/package.json b/package.json
@@ -11,25 +11,26 @@
     "cdk": "cdk"
   },
   "devDependencies": {
-    "@aws-cdk/assert": "^1.36.1",
+    "@aws-cdk/assert": "^1.38.0",
     "@types/jest": "^25.2.1",
-    "@types/node": "13.13.4",
-    "aws-cdk": "^1.36.1",
+    "@types/node": "^13.13.6",
+    "aws-cdk": "^1.38.0",
     "jest": "^25.5.2",
     "ts-jest": "^25.4.0",
     "ts-node": "^8.9.1",
     "typescript": "~3.8.3"
   },
   "dependencies": {
-    "@aws-cdk/aws-athena": "^1.36.1",
-    "@aws-cdk/aws-cloudformation": "^1.36.1",
-    "@aws-cdk/aws-ec2": "^1.36.1",
-    "@aws-cdk/aws-glue": "^1.36.1",
-    "@aws-cdk/aws-lakeformation": "^1.36.1",
-    "@aws-cdk/aws-lambda": "^1.36.1",
-    "@aws-cdk/aws-rds": "^1.36.1",
-    "@aws-cdk/aws-sagemaker": "^1.36.1",
-    "@aws-cdk/core": "^1.36.1",
+    "@aws-cdk/aws-athena": "^1.38.0",
+    "@aws-cdk/aws-cloudformation": "^1.38.0",
+    "@aws-cdk/aws-ec2": "^1.38.0",
+    "@aws-cdk/aws-glue": "^1.38.0",
+    "@aws-cdk/aws-lakeformation": "^1.38.0",
+    "@aws-cdk/aws-lambda": "^1.38.0",
+    "@aws-cdk/aws-rds": "^1.38.0",
+    "@aws-cdk/aws-sagemaker": "^1.38.0",
+    "@aws-cdk/core": "^1.38.0",
+    "@aws-cdk/custom-resources": "^1.38.0",
     "source-map-support": "^0.5.19",
     "update": "^0.7.4"
   }

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,6 @@`
`1`	`1`	`{`
`2`		`- "app": "npx ts-node bin/aws.ts"`
	`2`	`+ "app": "npx ts-node bin/aws.ts",`
	`3`	`+ "context": {`
	`4`	`+ "starterLakeFormationAdmin": "XXXX-this value gets populated by DeployChemblOpenTargetsEnv.sh script-XXXXX"`
	`5`	`+ }`
`3`	`6`	`}`
Original file line number	Diff line number	Diff line change
`@@ -42,7 +42,9 @@ export class ChemblStack extends DataSetStack{`
`42`	`42`	`"--DL_REGION": cdk.Stack.of(this).region,`
`43`	`43`	`"--GLUE_SRC_DATABASE": "chembl_25_src"`
`44`	`44`	`}`
`45`		`- });`
	`45`	`+ });`
	`46`	`+`
	`47`	`+`
`46`	`48`	`}`
`47`	`49`	`}`
`48`	`50`
Original file line number	Diff line number	Diff line change
`@@ -24,7 +24,7 @@ export class DataSetStack extends cdk.Stack {`
`24`	`24`	`this.DataLake = props.DataLake;`
`25`	`25`	`}`
`26`	`26`
`27`		`-`
	`27`	`+`
`28`	`28`	`public grantDatabasePermissions( principal: iam.IPrincipal, permissionGrant: DataLakeEnrollment.DatabasePermissionGrant){`
`29`	`29`	`this.Enrollment.grantDatabasePermission(principal, permissionGrant);`
`30`	`30`	`}`