Merge branch 'development' of ssh://git.amazon.com:2222/pkg/DataLakeAsCode into mainline

Author: paulu-aws

ApplyLakeFormationPermissions.sh

@@ -0,0 +1 @@
+python scripts/local.datalake.RemoveIamAllowedPrincipals.py

bin/aws.ts

@@ -2,40 +2,88 @@
 import 'source-map-support/register';
 import * as cdk from '@aws-cdk/core';
 import { BaselineStack } from '../lib/baseline-stack';
-import { DatalakeStack } from '../lib/datalake-stack';
+import { DataLakeStack } from '../lib/stacks/datalake-stack';
 import { OpenTargetsStack } from '../lib/opentargets-stack';
 import { ChemblStack } from '../lib/chembl-25-stack';
 import { AnalyticsStack } from '../lib/analytics-stack.js';
+import iam = require('@aws-cdk/aws-iam');
 import s3 = require('@aws-cdk/aws-s3');
-
+import { DataLakeEnrollment } from '../lib/constructs/data-lake-enrollment';
 
 const app = new cdk.App();
 const baseline = new BaselineStack(app, 'BaselineStack');
 
 
-const coreDataLake = new DatalakeStack(app, 'CoreDataLake', {
+const coreDataLake = new DataLakeStack(app, 'CoreDataLake', {
 
 });
 
-
-
 const chemblStack = new ChemblStack(app, 'ChemblStack', {
     database: baseline.ChemblDb,
     accessSecurityGroup: baseline.chemblDBChemblDbAccessSg,
     databaseSecret: baseline.chemblDBSecret,
-    dataLakeBucket: coreDataLake.DataLakeBucket
+    DataLake: coreDataLake
 });
 
 const openTargetsStack = new OpenTargetsStack(app, 'OpenTargetsStack', {
     sourceBucket: baseline.OpenTargetsSourceBucket,
     sourceBucketDataPrefix: '/opentargets/sourceExports/19.11/output/',
-    dataLakeBucket: coreDataLake.DataLakeBucket
+    DataLake: coreDataLake
 });
 
 const analyticsStack = new AnalyticsStack(app, 'AnalyticsStack', {
     targetVpc: baseline.Vpc,
 });
 
 
-chemblStack.grantRead(analyticsStack.NotebookRole);
-openTargetsStack.grantRead(analyticsStack.NotebookRole);
+chemblStack.grantIamRead(analyticsStack.NotebookRole);
+openTargetsStack.grantIamRead(analyticsStack.NotebookRole);
+
+
+
+
+
+
+
+
+const exampleUser = iam.User.fromUserName(coreDataLake, 'exampleGrantee', 'paul1' );
+
+var exampleGrant: DataLakeEnrollment.TablePermissionGrant = {
+    tables: ["association_data", "evidence_data","target_list","disease_list"],
+    DatabasePermissions: [DataLakeEnrollment.DatabasePermission.Alter, DataLakeEnrollment.DatabasePermission.CreateTable, DataLakeEnrollment.DatabasePermission.Drop],
+    GrantableDatabasePermissions: [DataLakeEnrollment.DatabasePermission.Alter, DataLakeEnrollment.DatabasePermission.CreateTable, DataLakeEnrollment.DatabasePermission.Drop],
+    TablePermissions: [DataLakeEnrollment.TablePermission.Select, DataLakeEnrollment.TablePermission.Insert, DataLakeEnrollment.TablePermission.Delete],
+    GrantableTablePermissions: [DataLakeEnrollment.TablePermission.Select]
+};
+
+openTargetsStack.grantTablePermissions(exampleUser, exampleGrant);
+
+
+
+
+// In the example below, we are using the compound_structures table from ChEMBL. It has the following table definition:
+// ['molregno', 'molfile', 'standard_inchi', 'standard_inchi_key', 'canonical_smiles']
+// Lets say we want to give a principal ONLY select permissions to everything in the compound_structures table BUT the 'canonical_smiles' column.
+
+var exampleTableWithColumnsGrant: DataLakeEnrollment.TableWithColumnPermissionGrant = {
+    table: "chembl_25_public_compound_structures",
+    // Note that we are NOT including 'canonical_smiles'. That effectivley prevents this user from querying that column.
+    columns: ['molregno', 'molfile', 'standard_inchi', 'standard_inchi_key'],
+    DatabasePermissions: [],
+    GrantableDatabasePermissions: [],
+    TableColumnPermissions: [DataLakeEnrollment.TablePermission.Select],
+    GrantableTableColumnPermissions: []
+};
+
+var exampleTableWithColumnsGrant_WithWildCard: DataLakeEnrollment.TableWithColumnPermissionGrant = {
+    table: "chembl_25_public_compound_structures",
+    wildCardFilter: DataLakeEnrollment.TableWithColumnFilter.Exclude,
+    columns: ['canonical_smiles'],
+    DatabasePermissions: [],
+    GrantableDatabasePermissions: [],
+    TableColumnPermissions: [DataLakeEnrollment.TablePermission.Select],
+    GrantableTableColumnPermissions: []
+};
+
+// Note that exampleTableWithColumnsGrant exampleTableWithColumnsGrant_WithWildCard grants the same effecitve permissions. One just uses a the wildcard.
+chemblStack.grantTableWithColumnPermissions(exampleUser, exampleTableWithColumnsGrant);

lib/analytics-stack.ts

@@ -6,7 +6,7 @@ import glue = require('@aws-cdk/aws-glue');
 import s3 = require('@aws-cdk/aws-s3');
 import s3assets = require('@aws-cdk/aws-s3-assets');
 import sagemaker = require('@aws-cdk/aws-sagemaker');
-import { DataSetStack, DataSetStackProps} from './dataset-stack';
+import { DataSetStack, DataSetStackProps} from './stacks/dataset-stack';
 
 export interface AnalyticsStackProps extends cdk.StackProps{
     targetVpc: ec2.Vpc

lib/chembl-25-stack.ts

@@ -5,15 +5,16 @@ import rds = require('@aws-cdk/aws-rds');
 import glue = require('@aws-cdk/aws-glue');
 import s3 = require('@aws-cdk/aws-s3');
 import s3assets = require('@aws-cdk/aws-s3-assets');
-import { RDSdataSetSetEnrollmentProps, RDSPostgresDataSetEnrollment } from './rds-data-set-enrollment';
-import { DataSetStack, DataSetStackProps} from './dataset-stack';
+import { RDSdataSetSetEnrollmentProps, RDSPostgresDataSetEnrollment } from './constructs/rds-data-set-enrollment';
+import { DataSetStack, DataSetStackProps} from './stacks/dataset-stack';
+
+
 
 
 export interface ChemblStackEnrollmentProps extends DataSetStackProps {
 	databaseSecret: rds.DatabaseSecret;
 	database: rds.DatabaseInstance;
 	accessSecurityGroup: ec2.SecurityGroup;
-	dataLakeBucket: s3.Bucket;
 }
 
 export class ChemblStack extends DataSetStack{
@@ -28,15 +29,15 @@ export class ChemblStack extends DataSetStack{
 	    	databaseSecret: props.databaseSecret,
 	    	database: props.database,
 	    	accessSecurityGroup: props.accessSecurityGroup,
-	    	dataLakeBucket: props.dataLakeBucket,
+	    	dataLakeBucket: props.DataLake.DataLakeBucket,
 	    	DataSetName: dataSetName,
 	    	JdbcTargetIncludePaths: ["chembl_25/%"],
 	    	GlueScriptPath: "scripts/glue.s3importchembl25.py",
 			GlueScriptArguments: {
 				"--job-language": "python", 
 				"--job-bookmark-option": "job-bookmark-disable",
 				"--enable-metrics": "",
-				"--DL_BUCKET": props.dataLakeBucket.bucketName,
+				"--DL_BUCKET": props.DataLake.DataLakeBucket.bucketName,
 				"--DL_PREFIX": "/"+dataSetName+"/",
 				"--DL_REGION": cdk.Stack.of(this).region,
 				"--GLUE_SRC_DATABASE": "chembl_25_src"