Adding notebook permissions and some additional inheritance

Author: EC2 Default User

bin/aws.ts

@@ -34,6 +34,13 @@ const openTargetsStack = new OpenTargetsStack(app, 'OpenTargetsStack', {
 });
 
 const analyticsStack = new AnalyticsStack(app, 'AnalyticsStack', {
-    targetVpc: baseline.Vpc
+    targetVpc: baseline.Vpc,
 });
 
+
+chemblStack.grantRead(analyticsStack.NotebookRole);
+openTargetsStack.grantRead(analyticsStack.NotebookRole);
+
+
+// chemblStack.grantRead(analyticsStack.NotebookRole);
+// openTargetsStack.grantRead(analyticsStack.NotebookRole);

lib/analytics-stack.ts

@@ -6,6 +6,7 @@ import glue = require('@aws-cdk/aws-glue');
 import s3 = require('@aws-cdk/aws-s3');
 import s3assets = require('@aws-cdk/aws-s3-assets');
 import sagemaker = require('@aws-cdk/aws-sagemaker');
+import { DataSetStack, DataSetStackProps} from './dataset-stack';
 
 export interface AnalyticsStackProps extends cdk.StackProps{
     targetVpc: ec2.Vpc
@@ -14,6 +15,8 @@ export interface AnalyticsStackProps extends cdk.StackProps{
 
 export class AnalyticsStack extends cdk.Stack {
 
+  public readonly NotebookRole: iam.Role;
+  
   constructor(scope: cdk.Construct, id: string, props: AnalyticsStackProps) {
     super(scope, id, props);
 
@@ -22,11 +25,16 @@ export class AnalyticsStack extends cdk.Stack {
        vpc: props.targetVpc
     });
 
+    const athenaSagingDirectory = new s3.Bucket(this, 'athenaStagingDir', {});
+    
+    
 
     const lifecycleCode = [
             {"content": cdk.Fn.base64(`
             wget -O /home/ec2-user/SageMaker/opentargets.chembl.example.ipynb https://raw.githubusercontent.com/paulu-aws/chembl-opentargets-data-lake-example/master/scripts/sagemaker.opentargets.chembl.example.ipynb
-            chown ec2-user /home/ec2-user/SageMaker/opentargets.chembl.example.ipynb
+            sudo chown ec2-user /home/ec2-user/SageMaker/opentargets.chembl.example.ipynb
+            sed -i 's/XXXXAthenaStagingDirectoryXXXX/${athenaSagingDirectory.bucketName}/g' opentargets.chembl.example.ipynb
+            sed -i 's/XXXXAthenaRegionXXXX/${cdk.Stack.of(this).region}/g' opentargets.chembl.example.ipynb
             `) }
         ];
     const sageMakerIntanceLifecyclePolicy = new sagemaker.CfnNotebookInstanceLifecycleConfig(this, 'notebookLifecyclePolicy', {
@@ -35,19 +43,20 @@ export class AnalyticsStack extends cdk.Stack {
 
     });
 
-    const  sageMakerInstanceRole = new iam.Role(this, 'notebookInstanceRole', {
+    this.NotebookRole = new iam.Role(this, 'notebookInstanceRole', {
         roleName: "chemblOpenTargetsNotebookRole",
         assumedBy: new iam.ServicePrincipal('sagemaker')
     });
 
+    athenaSagingDirectory.grantReadWrite(this.NotebookRole)
 
     new sagemaker.CfnNotebookInstance(this, 'analyticsNotebook', {
         instanceType : 'ml.t2.medium',
         volumeSizeInGb: 100,
         securityGroupIds: [notebookSg.securityGroupId],
         subnetId: props.targetVpc.selectSubnets({subnetType: ec2.SubnetType.PRIVATE}).subnetIds[0],
         notebookInstanceName: "Chembl-OpenTargets-Demo-Notebook",
-        roleArn: sageMakerInstanceRole.roleArn,
+        roleArn: this.NotebookRole.roleArn,
         directInternetAccess: 'Disabled',
         lifecycleConfigName: sageMakerIntanceLifecyclePolicy.notebookInstanceLifecycleConfigName
     });

lib/chembl-25-stack.ts

@@ -6,23 +6,25 @@ import glue = require('@aws-cdk/aws-glue');
 import s3 = require('@aws-cdk/aws-s3');
 import s3assets = require('@aws-cdk/aws-s3-assets');
 import { RDSdataSetSetEnrollmentProps, RDSPostgresDataSetEnrollment } from './rds-data-set-enrollment';
+import { DataSetStack, DataSetStackProps} from './dataset-stack';
 
 
-export interface ChemblStackEnrollmentProps extends cdk.StackProps {
+export interface ChemblStackEnrollmentProps extends DataSetStackProps {
 	databaseSecret: rds.DatabaseSecret;
 	database: rds.DatabaseInstance;
 	accessSecurityGroup: ec2.SecurityGroup;
 	dataLakeBucket: s3.Bucket;
 }
 
-export class ChemblStack extends cdk.Stack{
+export class ChemblStack extends DataSetStack{
 	constructor(scope: cdk.Construct, id: string, props: ChemblStackEnrollmentProps) {
 		super(scope, id, props);
 
 
 		const dataSetName = "chembl_25";
 
-	    const chembl25 = new RDSPostgresDataSetEnrollment(this, 'chembl-25-enrollment', {
+		
+		this.Enrollment = new RDSPostgresDataSetEnrollment(this, 'chembl-25-enrollment', {
 	    	databaseSecret: props.databaseSecret,
 	    	database: props.database,
 	    	accessSecurityGroup: props.accessSecurityGroup,

lib/data-set-enrollment.ts

@@ -34,6 +34,11 @@ export class DataSetEnrollment extends cdk.Construct {
     public readonly DataSetGlueRole: iam.Role;
     public readonly Dataset_Source: glue.Database;
     public readonly Dataset_Datalake: glue.Database;
+    
+    public readonly DataLakeBucketName: string;
+    public readonly DataLakePrefix: string;
+
+
 
 	private setupCrawler(targetGlueDatabase: glue.Database, targets: glue.CfnCrawler.TargetsProperty, isSourceCrawler: boolean){
 
@@ -57,6 +62,10 @@ export class DataSetEnrollment extends cdk.Construct {
 	constructor(scope: cdk.Construct, id: string, props: DataSetEnrollmentProps) {
 		super(scope, id);	
 
+		
+		this.DataLakeBucketName	= props.GlueScriptArguments['--DL_BUCKET'];
+		this.DataLakePrefix = props.GlueScriptArguments['--DL_PREFIX'];
+		
 		this.DataSetName = props.dataSetName;
 
 		this.Dataset_Source = new glue.Database(this, `${props.dataSetName}_src`, {

lib/datalake-stack.ts

@@ -43,6 +43,33 @@ export class DataLakeEnrollment extends cdk.Construct {
   constructor(scope: cdk.Construct, id: string, props: DataLakeEnrollmentProps) {
     super(scope, id);
   }
+  
+	public grantRead(principal: iam.Role){
+		
+		
+		const dataLakeBucket = s3.Bucket.fromBucketName(this, 'dataLakeBucket', this.DataEnrollment.DataLakeBucketName);
+		dataLakeBucket.grantRead(principal, this.DataEnrollment.DataLakePrefix + "*")
+		
+		const gluePolicy = new iam.PolicyStatement({
+	        actions: ["glue:GetDatabase"],
+	        effect: iam.Effect.ALLOW,
+	        resources: [`arn:aws:glue:${cdk.Stack.of(this).region}:${cdk.Stack.of(this).account}:catalog`, 
+	        			`arn:aws:glue:${cdk.Stack.of(this).region}:${cdk.Stack.of(this).account}:database/default`, 
+	        			this.DataEnrollment.Dataset_Datalake.databaseArn
+	        		   ]
+	    });    
+		
+		const athenaPolicy = new iam.PolicyStatement({
+	        actions: ["athena:*"],
+	        effect: iam.Effect.ALLOW,
+	        resources: ["*"],
+	    });    
+	    
+	    principal.addToPolicy(gluePolicy);
+	    principal.addToPolicy(athenaPolicy);
+		
+	
+	}
 }
 
 

lib/dataset-stack.ts

@@ -0,0 +1,28 @@
+import * as cdk from '@aws-cdk/core';
+import ec2 = require('@aws-cdk/aws-ec2');
+import iam = require('@aws-cdk/aws-iam');
+import rds = require('@aws-cdk/aws-rds');
+import glue = require('@aws-cdk/aws-glue');
+import s3 = require('@aws-cdk/aws-s3');
+import s3assets = require('@aws-cdk/aws-s3-assets');
+import { DataSetEnrollmentProps, DataSetEnrollment } from './data-set-enrollment';
+import { DataLakeEnrollmentProps, DataLakeEnrollment} from './datalake-stack';
+
+	
+export interface DataSetStackProps extends cdk.StackProps {
+	dataLakeBucket: s3.Bucket;	
+}	
+	
+
+export class DataSetStack extends cdk.Stack {
+  
+  public Enrollment: DataLakeEnrollment;
+  
+  constructor(scope: cdk.Construct, id: string, props: DataSetStackProps) {
+    super(scope, id, props);
+  }
+  
+  public grantRead(role: iam.Role){
+      this.Enrollment.grantRead(role);
+  }
+}

lib/opentargets-stack.ts

@@ -5,23 +5,23 @@ import glue = require('@aws-cdk/aws-glue');
 import s3 = require('@aws-cdk/aws-s3');
 import s3assets = require('@aws-cdk/aws-s3-assets');
 import { S3dataSetEnrollmentProps, S3dataSetEnrollment } from './s3-data-set-enrollment';
+import { DataSetStack, DataSetStackProps} from './dataset-stack';
 
 
-
-export interface OpenTargetsEnrollmentProps extends cdk.StackProps {
+export interface OpenTargetsEnrollmentProps extends DataSetStackProps {
     sourceBucket: s3.IBucket;
     sourceBucketDataPrefix: string;
 	dataLakeBucket: s3.Bucket;
 }
 
 
-export class OpenTargetsStack extends cdk.Stack{
+export class OpenTargetsStack extends DataSetStack{
 
 	constructor(scope: cdk.Construct, id: string, props: OpenTargetsEnrollmentProps) {
 	    super(scope, id, props);
 
 
-	    const openTargets1911 = new S3dataSetEnrollment(this, 'openTargets-1911-enrollment', {
+	    this.Enrollment = new S3dataSetEnrollment(this, 'openTargets-1911-enrollment', {
 	        DataSetName: "opentargets_1911",
 	        sourceBucket: props.sourceBucket,
             sourceBucketDataPrefixes: [

scripts/sagemaker.opentargets.chembl.example.ipynb

@@ -6,13 +6,13 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "#import sys\n",
-    "#!{sys.executable} -m pip install PyAthena"
+    "import sys\n",
+    "!{sys.executable} -m pip install PyAthena"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -28,8 +28,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "conn = connect(s3_staging_dir='s3://chembl-opentargets-43762-3/athena/',\n",
-    "               region_name='us-east-1')"
+    "conn = connect(s3_staging_dir='s3://XXXXAthenaStagingDirectoryXXXX/athena/',\n",
+    "               region_name='XXXXAthenaRegionXXXX')"
    ]
   },
   {