Skip to content

Commit a48cea0

Browse files
paulu-awspaulu-aws
committed
Merge branch 'mainline' of ssh://git.amazon.com:2222/pkg/DataLakeAsCode into mainline
2 parents a630978 + 78cd7ba commit a48cea0

File tree

10 files changed

+187
-68
lines changed

10 files changed

+187
-68
lines changed

‎DeployChemblOpenTargetsEnv.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
1+
#!/bin/sh
12
npm run build
23
cdk bootstrap
34
cdk deploy BaselineStack --require-approval never
5+
currentPrincipalArn=$(aws sts get-caller-identity --query Arn --output text)
6+
jq '.context.starterLakeFormationAdmin = $currentPrincipalArn' --arg currentPrincipalArn $currentPrincipalArn cdk.json > tmp.$$.json && mv tmp.$$.json cdk.json
47
cdk deploy CoreDataLake --require-approval never
58
cdk deploy ChemblStack --require-approval never
69
cdk deploy OpenTargetsStack --require-approval never

‎bin/aws.ts‎

Lines changed: 36 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ const baseline = new BaselineStack(app, 'BaselineStack');
1515

1616

1717
const coreDataLake = new DataLakeStack(app, 'CoreDataLake', {
18-
18+
starterLakeFormationAdminPrincipalArn: app.node.tryGetContext("starterLakeFormationAdmin")
1919
});
2020

2121
const chemblStack = new ChemblStack(app, 'ChemblStack', {
@@ -36,6 +36,9 @@ const analyticsStack = new AnalyticsStack(app, 'AnalyticsStack', {
3636
});
3737

3838

39+
40+
41+
3942
chemblStack.grantIamRead(analyticsStack.NotebookRole);
4043
openTargetsStack.grantIamRead(analyticsStack.NotebookRole);
4144

@@ -46,44 +49,44 @@ openTargetsStack.grantIamRead(analyticsStack.NotebookRole);
4649

4750

4851

49-
const exampleUser = iam.User.fromUserName(coreDataLake, 'exampleGrantee', 'paul1' );
52+
// const exampleUser = iam.User.fromUserName(coreDataLake, 'exampleGrantee', 'paul1' );
5053

51-
var exampleGrant: DataLakeEnrollment.TablePermissionGrant = {
52-
tables: ["association_data", "evidence_data","target_list","disease_list"],
53-
DatabasePermissions: [DataLakeEnrollment.DatabasePermission.Alter, DataLakeEnrollment.DatabasePermission.CreateTable, DataLakeEnrollment.DatabasePermission.Drop],
54-
GrantableDatabasePermissions: [DataLakeEnrollment.DatabasePermission.Alter, DataLakeEnrollment.DatabasePermission.CreateTable, DataLakeEnrollment.DatabasePermission.Drop],
55-
TablePermissions: [DataLakeEnrollment.TablePermission.Select, DataLakeEnrollment.TablePermission.Insert, DataLakeEnrollment.TablePermission.Delete],
56-
GrantableTablePermissions: [DataLakeEnrollment.TablePermission.Select]
57-
};
54+
// var exampleGrant: DataLakeEnrollment.TablePermissionGrant = {
55+
// tables: ["association_data", "evidence_data","target_list","disease_list"],
56+
// DatabasePermissions: [DataLakeEnrollment.DatabasePermission.Alter, DataLakeEnrollment.DatabasePermission.CreateTable, DataLakeEnrollment.DatabasePermission.Drop],
57+
// GrantableDatabasePermissions: [DataLakeEnrollment.DatabasePermission.Alter, DataLakeEnrollment.DatabasePermission.CreateTable, DataLakeEnrollment.DatabasePermission.Drop],
58+
// TablePermissions: [DataLakeEnrollment.TablePermission.Select, DataLakeEnrollment.TablePermission.Insert, DataLakeEnrollment.TablePermission.Delete],
59+
// GrantableTablePermissions: [DataLakeEnrollment.TablePermission.Select]
60+
// };
5861

59-
openTargetsStack.grantTablePermissions(exampleUser, exampleGrant);
62+
// openTargetsStack.grantTablePermissions(exampleUser, exampleGrant);
6063

6164

6265

6366

64-
// In the example below, we are using the compound_structures table from ChEMBL. It has the following table definition:
65-
// ['molregno', 'molfile', 'standard_inchi', 'standard_inchi_key', 'canonical_smiles']
66-
// Lets say we want to give a principal ONLY select permissions to everything in the compound_structures table BUT the 'canonical_smiles' column.
67+
// // In the example below, we are using the compound_structures table from ChEMBL. It has the following table definition:
68+
// // ['molregno', 'molfile', 'standard_inchi', 'standard_inchi_key', 'canonical_smiles']
69+
// // Lets say we want to give a principal ONLY select permissions to everything in the compound_structures table BUT the 'canonical_smiles' column.
6770

68-
var exampleTableWithColumnsGrant: DataLakeEnrollment.TableWithColumnPermissionGrant = {
69-
table: "chembl_25_public_compound_structures",
70-
// Note that we are NOT including 'canonical_smiles'. That effectivley prevents this user from querying that column.
71-
columns: ['molregno', 'molfile', 'standard_inchi', 'standard_inchi_key'],
72-
DatabasePermissions: [],
73-
GrantableDatabasePermissions: [],
74-
TableColumnPermissions: [DataLakeEnrollment.TablePermission.Select],
75-
GrantableTableColumnPermissions: []
76-
};
71+
// var exampleTableWithColumnsGrant: DataLakeEnrollment.TableWithColumnPermissionGrant = {
72+
// table: "chembl_25_public_compound_structures",
73+
// // Note that we are NOT including 'canonical_smiles'. That effectivley prevents this user from querying that column.
74+
// columns: ['molregno', 'molfile', 'standard_inchi', 'standard_inchi_key'],
75+
// DatabasePermissions: [],
76+
// GrantableDatabasePermissions: [],
77+
// TableColumnPermissions: [DataLakeEnrollment.TablePermission.Select],
78+
// GrantableTableColumnPermissions: []
79+
// };
7780

78-
var exampleTableWithColumnsGrant_WithWildCard: DataLakeEnrollment.TableWithColumnPermissionGrant = {
79-
table: "chembl_25_public_compound_structures",
80-
wildCardFilter: DataLakeEnrollment.TableWithColumnFilter.Exclude,
81-
columns: ['canonical_smiles'],
82-
DatabasePermissions: [],
83-
GrantableDatabasePermissions: [],
84-
TableColumnPermissions: [DataLakeEnrollment.TablePermission.Select],
85-
GrantableTableColumnPermissions: []
86-
};
81+
// var exampleTableWithColumnsGrant_WithWildCard: DataLakeEnrollment.TableWithColumnPermissionGrant = {
82+
// table: "chembl_25_public_compound_structures",
83+
// wildCardFilter: DataLakeEnrollment.TableWithColumnFilter.Exclude,
84+
// columns: ['canonical_smiles'],
85+
// DatabasePermissions: [],
86+
// GrantableDatabasePermissions: [],
87+
// TableColumnPermissions: [DataLakeEnrollment.TablePermission.Select],
88+
// GrantableTableColumnPermissions: []
89+
// };
8790

88-
// Note that exampleTableWithColumnsGrant exampleTableWithColumnsGrant_WithWildCard grants the same effecitve permissions. One just uses a the wildcard.
89-
chemblStack.grantTableWithColumnPermissions(exampleUser, exampleTableWithColumnsGrant);
91+
// // Note that exampleTableWithColumnsGrant exampleTableWithColumnsGrant_WithWildCard grants the same effecitve permissions. One just uses a the wildcard.
92+
// chemblStack.grantTableWithColumnPermissions(exampleUser, exampleTableWithColumnsGrant);

‎cdk.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
11
{
2-
"app": "npx ts-node bin/aws.ts"
2+
"app": "npx ts-node bin/aws.ts",
3+
"context": {
4+
"starterLakeFormationAdmin": "XXXX-this value gets populated by DeployChemblOpenTargetsEnv.sh script-XXXXX"
5+
}
36
}

‎lib/chembl-25-stack.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,9 @@ export class ChemblStack extends DataSetStack{
4242
"--DL_REGION": cdk.Stack.of(this).region,
4343
"--GLUE_SRC_DATABASE": "chembl_25_src"
4444
}
45-
});
45+
});
46+
47+
4648
}
4749
}
4850

‎lib/constructs/data-lake-enrollment.ts‎

Lines changed: 85 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -13,20 +13,34 @@ import { DataSetEnrollmentProps, DataSetEnrollment } from './data-set-enrollment
1313

1414
export class DataLakeEnrollment extends cdk.Construct {
1515

16-
public DataEnrollment: DataSetEnrollment;
17-
public DataSetName: string;
18-
private CoarseAthenaAccessPolicy: iam.ManagedPolicy;
19-
private CoarseResourceAccessPolicy: iam.ManagedPolicy;
20-
private CoarseIamPolciesApplied: boolean;
16+
public DataEnrollment: DataSetEnrollment;
17+
public DataSetName: string;
18+
private CoarseAthenaAccessPolicy: iam.ManagedPolicy;
19+
private CoarseResourceAccessPolicy: iam.ManagedPolicy;
20+
private CoarseIamPolciesApplied: boolean;
2121

22-
constructor(scope: cdk.Construct, id: string, props: DataLakeEnrollment.DataLakeEnrollmentProps) {
22+
constructor(scope: cdk.Construct, id: string, props: DataLakeEnrollment.DataLakeEnrollmentProps) {
2323
super(scope, id);
2424

2525

2626
this.DataSetName = props.DataSetName;
2727
this.CoarseIamPolciesApplied = false;
2828

2929
}
30+
31+
protected grantGlueRoleLakeFormationPermissions(DataSetGlueRole: iam.Role, DataSetName: string) {
32+
33+
this.grantDataLocationPermissions(this.DataEnrollment.DataSetGlueRole, {
34+
Grantable: true,
35+
GrantResourcePrefix: `${DataSetName}locationGrant`
36+
});
37+
this.grantDatabasePermission(this.DataEnrollment.DataSetGlueRole, {
38+
DatabasePermissions: [DataLakeEnrollment.DatabasePermission.All],
39+
GrantableDatabasePermissions: [DataLakeEnrollment.DatabasePermission.All],
40+
GrantResourcePrefix: `${DataSetName}RoleGrant`
41+
}, true);
42+
}
43+
3044

3145
public createCoarseIamPolicy(){
3246

@@ -225,6 +239,45 @@ export class DataLakeEnrollment extends cdk.Construct {
225239

226240
}
227241

242+
public grantDataLocationPermissions(principal: iam.IPrincipal, permissionGrant: DataLakeEnrollment.DataLocationGrant){
243+
244+
var grantIdPrefix = ""
245+
var dataLakePrincipal : lakeformation.CfnPermissions.DataLakePrincipalProperty = {
246+
dataLakePrincipalIdentifier: ""
247+
};
248+
249+
var dataLocationProperty : lakeformation.CfnPermissions.ResourceProperty = {
250+
dataLocationResource: {
251+
s3Resource: `arn:aws:s3:::${this.DataEnrollment.DataLakeBucketName}${this.DataEnrollment.DataLakePrefix}`
252+
}
253+
};
254+
const resolvedPrincipalType = this.determinePrincipalType(principal);
255+
256+
if(resolvedPrincipalType === iam.Role) {
257+
const resolvedPrincipal = principal as iam.Role;
258+
259+
if(permissionGrant.GrantResourcePrefix){
260+
grantIdPrefix = `${permissionGrant.GrantResourcePrefix}-${this.DataSetName}`
261+
}else{
262+
grantIdPrefix = `${resolvedPrincipal.roleName}-${this.DataSetName}`
263+
}
264+
dataLakePrincipal = { dataLakePrincipalIdentifier: resolvedPrincipal.roleArn };
265+
}
266+
267+
if(resolvedPrincipalType === iam.User){
268+
const resolvedPrincipal = principal as iam.User;
269+
grantIdPrefix = `${resolvedPrincipal.userName}-${this.DataSetName}`
270+
dataLakePrincipal = { dataLakePrincipalIdentifier: resolvedPrincipal.userArn };
271+
}
272+
273+
if(permissionGrant.Grantable){
274+
this.createLakeFormationPermission(`${grantIdPrefix}-locationGrant`,dataLakePrincipal , dataLocationProperty, ['DATA_LOCATION_ACCESS'], ['DATA_LOCATION_ACCESS']);
275+
}else {
276+
this.createLakeFormationPermission(`${grantIdPrefix}-locationGrant`,dataLakePrincipal , dataLocationProperty, ['DATA_LOCATION_ACCESS'], ['']);
277+
}
278+
279+
280+
}
228281

229282
public grantTableWithColumnPermissions(principal: iam.IPrincipal, permissionGrant: DataLakeEnrollment.TableWithColumnPermissionGrant){
230283

@@ -277,23 +330,27 @@ export class DataLakeEnrollment extends cdk.Construct {
277330

278331
}
279332

280-
public grantDatabasePermission(principal: iam.IPrincipal, permissionGrant: DataLakeEnrollment.DatabasePermissionGrant){
333+
public grantDatabasePermission(principal: iam.IPrincipal, permissionGrant: DataLakeEnrollment.DatabasePermissionGrant, includeSourceDb: boolean = false){
281334

282335

283336
var grantIdPrefix = ""
284337
var dataLakePrincipal : lakeformation.CfnPermissions.DataLakePrincipalProperty = {
285338
dataLakePrincipalIdentifier: ""
286339
};
287-
var databaseResourceProperty : lakeformation.CfnPermissions.ResourceProperty = {
288-
//dataLocationResource: {resourceArn: this.DataEnrollment.DataLakeBucketName},
340+
var databaseResourceProperty : lakeformation.CfnPermissions.ResourceProperty = {
289341
databaseResource: {name: this.DataEnrollment.Dataset_Datalake.databaseName}
290342
};
291343

292344
const resolvedPrincipalType = this.determinePrincipalType(principal);
293345

294346
if(resolvedPrincipalType === iam.Role) {
295347
const resolvedPrincipal = principal as iam.Role;
296-
grantIdPrefix = `${resolvedPrincipal.roleArn}-${this.DataSetName}`
348+
349+
if(permissionGrant.GrantResourcePrefix){
350+
grantIdPrefix = `${permissionGrant.GrantResourcePrefix}-${this.DataSetName}`
351+
}else{
352+
grantIdPrefix = `${resolvedPrincipal.roleName}-${this.DataSetName}`
353+
}
297354
dataLakePrincipal = { dataLakePrincipalIdentifier: resolvedPrincipal.roleArn };
298355
}
299356

@@ -305,11 +362,23 @@ export class DataLakeEnrollment extends cdk.Construct {
305362

306363
this.createLakeFormationPermission(`${grantIdPrefix}-databaseGrant`,dataLakePrincipal , databaseResourceProperty, permissionGrant.DatabasePermissions, permissionGrant.GrantableDatabasePermissions)
307364

365+
if(includeSourceDb){
366+
367+
databaseResourceProperty = {
368+
//dataLocationResource: {resourceArn: this.DataEnrollment.DataLakeBucketName},
369+
databaseResource: {name: this.DataEnrollment.Dataset_Source.databaseName}
370+
};
371+
372+
this.createLakeFormationPermission(`${grantIdPrefix}-databaseSrcGrant`,dataLakePrincipal , databaseResourceProperty, permissionGrant.DatabasePermissions, permissionGrant.GrantableDatabasePermissions)
373+
374+
}
375+
376+
308377
}
309378

310379

311380
public grantTablePermissions(principal: iam.IPrincipal, permissionGrant: DataLakeEnrollment.TablePermissionGrant){
312-
381+
313382
const coreGrant = this.setupIamAndLakeFormationDatabasePermissionForPrincipal(principal, permissionGrant.DatabasePermissions, permissionGrant.GrantableDatabasePermissions);
314383

315384
permissionGrant.tables.forEach(table => {
@@ -462,8 +531,13 @@ export namespace DataLakeEnrollment
462531
export interface DatabasePermissionGrant {
463532
DatabasePermissions: Array<DatabasePermission>;
464533
GrantableDatabasePermissions: Array<DatabasePermission>;
534+
GrantResourcePrefix?: string;
465535
}
466536

537+
export interface DataLocationGrant{
538+
Grantable: boolean;
539+
GrantResourcePrefix?: string;
540+
}
467541

468542
export interface TablePermissionGrant {
469543
tables: Array<string>;

‎lib/constructs/rds-data-set-enrollment.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ export class RDSPostgresDataSetEnrollment extends DataLakeEnrollment {
3434
});
3535

3636
}
37-
37+
3838
this.DataEnrollment = new DataSetEnrollment(this, 'rdsDatasetEnrollment', {
3939
dataLakeBucket: props.dataLakeBucket,
4040
dataSetName: dataSetName,
@@ -61,7 +61,10 @@ export class RDSPostgresDataSetEnrollment extends DataLakeEnrollment {
6161
GlueScriptArguments: props.GlueScriptArguments
6262

6363
});
64-
64+
65+
6566
this.createCoarseIamPolicy();
67+
this.grantGlueRoleLakeFormationPermissions(this.DataEnrollment.DataSetGlueRole, props.DataSetName);
68+
6669
}
6770
}

‎lib/constructs/s3-data-set-enrollment.ts‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,22 @@ export interface S3dataSetEnrollmentProps extends DataLakeEnrollment.DataLakeEnr
1616

1717

1818
export class S3dataSetEnrollment extends DataLakeEnrollment{
19+
20+
21+
22+
grantGlueRoleLakeFormationPermissions(DataSetGlueRole: iam.Role, DataSetName: string) {
23+
24+
this.grantDataLocationPermissions(this.DataEnrollment.DataSetGlueRole, {
25+
Grantable: true,
26+
GrantResourcePrefix: `${DataSetName}locationGrant`
27+
});
28+
this.grantDatabasePermission(this.DataEnrollment.DataSetGlueRole, {
29+
DatabasePermissions: [DataLakeEnrollment.DatabasePermission.All],
30+
GrantableDatabasePermissions: [DataLakeEnrollment.DatabasePermission.All],
31+
GrantResourcePrefix: `${DataSetName}RoleGrant`
32+
}, true);
33+
}
34+
1935
constructor(scope: cdk.Construct, id: string, props: S3dataSetEnrollmentProps) {
2036
super(scope, id, props);
2137

@@ -47,7 +63,7 @@ export class S3dataSetEnrollment extends DataLakeEnrollment{
4763

4864
}
4965

50-
this.DataEnrollment = new DataSetEnrollment(this, 'openTargetsEnrollment', {
66+
this.DataEnrollment = new DataSetEnrollment(this, `${props.DataSetName}-s3Enrollment`, {
5167
dataLakeBucket: props.dataLakeBucket,
5268
dataSetName: dataSetName,
5369
SourceAccessPolicy: s3AccessPolicy,
@@ -58,6 +74,12 @@ export class S3dataSetEnrollment extends DataLakeEnrollment{
5874
GlueScriptArguments: props.GlueScriptArguments
5975
});
6076

61-
this.createCoarseIamPolicy();
77+
this.createCoarseIamPolicy();
78+
79+
this.grantGlueRoleLakeFormationPermissions(this.DataEnrollment.DataSetGlueRole, props.DataSetName);
80+
81+
82+
83+
6284
}
6385
}

‎lib/stacks/datalake-stack.ts‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,15 @@ import cfn = require("@aws-cdk/aws-cloudformation");
88
import s3 = require('@aws-cdk/aws-s3');
99
import s3assets = require('@aws-cdk/aws-s3-assets');
1010
import lakeformation = require('@aws-cdk/aws-lakeformation');
11-
import lambda = require('@aws-cdk/aws-lambda')
11+
import { CustomResource } from '@aws-cdk/core';
12+
import * as cr from '@aws-cdk/custom-resources';
13+
import lambda = require('@aws-cdk/aws-lambda');
1214
import fs = require('fs');
1315

1416
import { DataSetEnrollmentProps, DataSetEnrollment } from '../constructs/data-set-enrollment';
1517

1618
export interface DataLakeStackProps extends cdk.StackProps {
17-
19+
starterLakeFormationAdminPrincipalArn: string;
1820
}
1921

2022
export class DataLakeStack extends cdk.Stack {
@@ -25,6 +27,7 @@ export class DataLakeStack extends cdk.Stack {
2527
public readonly LakeFormationResource: lakeformation.CfnResource;
2628
public readonly PrimaryAthenaWorkgroup: athena.CfnWorkGroup;
2729
private readonly bucketRole: iam.Role;
30+
private readonly starterAdminArn: string;
2831

2932
public grantAthenaResultsBucketPermission(principal: iam.IPrincipal){
3033

@@ -63,9 +66,14 @@ export class DataLakeStack extends cdk.Stack {
6366
super(scope, id, props);
6467

6568
this.DataLakeBucket = new s3.Bucket(this, 'dataLakeBucket');
66-
this.AthenaResultsBucket = new s3.Bucket(this, 'athenaResultsBucket');
67-
68-
69+
this.AthenaResultsBucket = new s3.Bucket(this, 'athenaResultsBucket');
70+
71+
new lakeformation.CfnDataLakeSettings(this, 'starterAdminPermission', {
72+
admins: [{
73+
dataLakePrincipalIdentifier: props.starterLakeFormationAdminPrincipalArn
74+
}]
75+
});
76+
6977
const coarseAthenaResultBucketAccess = {
7078
"Version": "2012-10-17",
7179
"Statement": [

0 commit comments

Comments
 (0)