Skip to content

Commit 084641b

Browse files
JohannesKoningsJohannesKonings
committed
update
1 parent d04f381 commit 084641b

File tree

3 files changed

+57
-57
lines changed

3 files changed

+57
-57
lines changed

‎cdk/lib/cdk-stack.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -258,15 +258,16 @@ export class CdkStack extends Stack {
258258
new DdbExport(this, 'ddb-export', {
259259
name: name,
260260
table: table,
261-
bucket: firehoseBucket,
261+
firehoseBucket: firehoseBucket,
262+
athenaResultBucket: athenaQueryResults,
262263
glueDb: glueDb,
263264
athenaWorkgroup: athenaWorkgroup,
264265
})
265266

266267
new DdbExportStepFunction(this, 'ddb-export-step-function', {
267268
name: name,
268269
table: table,
269-
bucket: firehoseBucket,
270+
firehoseBucket: firehoseBucket,
270271
athenaResultBucket: athenaQueryResults,
271272
glueDb: glueDb,
272273
athenaWorkgroup: athenaWorkgroup,

‎cdk/lib/ddb-export/ddb-export-step-function.ts

Lines changed: 13 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import { join } from 'path'
1717
export interface DdbExportStepFunctionProps {
1818
name: string
1919
table: dynamodb.ITable
20-
bucket: s3.IBucket
20+
firehoseBucket: s3.IBucket
2121
athenaResultBucket: s3.IBucket
2222
glueDb: glueAlpha.IDatabase
2323
athenaWorkgroup: athena.CfnWorkGroup
@@ -33,7 +33,7 @@ export class DdbExportStepFunction extends Construct {
3333
environment: {
3434
REGION: Stack.of(this).region,
3535
DYNAMO_DB_TABLE_ARN: props.table.tableArn,
36-
S3_BUCKET_NAME: props.bucket.bucketName,
36+
S3_BUCKET_NAME: props.firehoseBucket.bucketName,
3737
},
3838
})
3939
lambdaStartExport.addToRolePolicy(
@@ -45,13 +45,13 @@ export class DdbExportStepFunction extends Construct {
4545
lambdaStartExport.addToRolePolicy(
4646
new iam.PolicyStatement({
4747
actions: ['s3:PutObject'],
48-
resources: [`${props.bucket.bucketArn}/*`],
48+
resources: [`${props.firehoseBucket.bucketArn}/*`],
4949
})
5050
)
5151
lambdaStartExport.addToRolePolicy(
5252
new iam.PolicyStatement({
5353
actions: ['kms:Decrypt'],
54-
resources: [props.bucket.encryptionKey!.keyArn],
54+
resources: [props.firehoseBucket.encryptionKey!.keyArn],
5555
})
5656
)
5757

@@ -61,7 +61,7 @@ export class DdbExportStepFunction extends Construct {
6161
environment: {
6262
REGION: Stack.of(this).region,
6363
DYNAMO_DB_TABLE_ARN: props.table.tableArn,
64-
S3_BUCKET_NAME: props.bucket.bucketName,
64+
S3_BUCKET_NAME: props.firehoseBucket.bucketName,
6565
},
6666
})
6767
lambdaCheckExportState.addToRolePolicy(
@@ -101,7 +101,7 @@ export class DdbExportStepFunction extends Construct {
101101

102102
const getSqlString = (file: string): string => {
103103
let createTableCommand = readFileSync(join(__dirname, `${file}`), 'utf-8').toString()
104-
const s3Location = `s3://${props.bucket.bucketName}/ddb-exports/AWSDynamoDB/ddb-export-id/data/`
104+
const s3Location = `s3://${props.firehoseBucket.bucketName}/ddb-exports/AWSDynamoDB/ddb-export-id/data/`
105105
createTableCommand = createTableCommand.replace(/s3Location/g, s3Location)
106106
createTableCommand = createTableCommand.replace(/table_name/g, athenaTableName)
107107
return createTableCommand
@@ -137,18 +137,14 @@ export class DdbExportStepFunction extends Construct {
137137
)
138138
lambdaCreateAthenaTable.addToRolePolicy(
139139
new iam.PolicyStatement({
140-
actions: ['s3:*'],
141-
resources: [`*`],
142-
// actions: ['s3:PutObject'],
143-
// resources: [`${props.bucket.bucketArn}/*`],
140+
actions: ['s3:PutObject'],
141+
resources: [props.athenaResultBucket.bucketArn, `${props.athenaResultBucket.bucketArn}/*`],
144142
})
145143
);
146144
lambdaCreateAthenaTable.addToRolePolicy(
147145
new iam.PolicyStatement({
148-
actions: ['kms:*'],
149-
resources: ['*'],
150-
// actions: ['kms:Decrypt'],
151-
// resources: [props.bucket.encryptionKey!.keyArn],
146+
actions: ['kms:Decrypt', 'kms:GenerateDataKey'],
147+
resources: [props.athenaResultBucket.encryptionKey!.keyArn],
152148
})
153149
);
154150
lambdaCreateAthenaTable.addToRolePolicy(
@@ -175,8 +171,8 @@ export class DdbExportStepFunction extends Construct {
175171
'glue:UpdateTable'
176172
],
177173
resources: [
178-
`arn:aws:glue:${Stack.of(this).region}:${Stack.of(this).account}:catalog`, // remove?
179-
`arn:aws:glue:${Stack.of(this).region}:${Stack.of(this).account}:database/default`, // remove?
174+
`arn:aws:glue:${Stack.of(this).region}:${Stack.of(this).account}:catalog`,
175+
// `arn:aws:glue:${Stack.of(this).region}:${Stack.of(this).account}:database/default`,
180176
props.glueDb.databaseArn,
181177
`arn:aws:glue:${Stack.of(this).region}:${Stack.of(this).account}:table/${props.glueDb.databaseName}/${athenaTableName}`],
182178
})
@@ -225,7 +221,7 @@ export class DdbExportStepFunction extends Construct {
225221
// https://aws.amazon.com/de/premiumsupport/knowledge-center/access-denied-athena/
226222
sfn.addToRolePolicy(
227223
new iam.PolicyStatement({
228-
actions: ['s3:*'],
224+
actions: ['s3:PutObject'],
229225
resources: [props.athenaResultBucket.bucketArn, `${props.athenaResultBucket.bucketArn}/*`],
230226
})
231227
)
@@ -235,31 +231,5 @@ export class DdbExportStepFunction extends Construct {
235231
resources: [props.athenaResultBucket.encryptionKey!.keyArn],
236232
})
237233
)
238-
// sfn.addToRolePolicy(
239-
// new iam.PolicyStatement({
240-
// actions: [
241-
// 'glue:BatchCreatePartition',
242-
// 'glue:BatchDeletePartition',
243-
// 'glue:BatchDeleteTable',
244-
// 'glue:BatchGetPartition',
245-
// 'glue:CreateDatabase',
246-
// 'glue:CreatePartition',
247-
// 'glue:CreateTable',
248-
// 'glue:DeleteDatabase',
249-
// 'glue:DeletePartition',
250-
// 'glue:DeleteTable',
251-
// 'glue:GetDatabase',
252-
// 'glue:GetDatabases',
253-
// 'glue:GetPartition',
254-
// 'glue:GetPartitions',
255-
// 'glue:GetTable',
256-
// 'glue:GetTables',
257-
// 'glue:UpdateDatabase',
258-
// 'glue:UpdatePartition',
259-
// 'glue:UpdateTable'
260-
// ],
261-
// resources: [props.glueDb.databaseArn, `arn:aws:glue:${Stack.of(this).region}:${Stack.of(this).account}:table/${props.glueDb.databaseName}/${athenaTableName}`],
262-
// })
263-
// )
264234
}
265235
}

‎cdk/lib/ddb-export/ddb-export.ts

Lines changed: 41 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@ import { join } from 'path'
77
export interface DdbExportProps {
88
name: string
99
table: dynamodb.ITable
10-
bucket: s3.IBucket
10+
firehoseBucket: s3.IBucket
11+
athenaResultBucket: s3.IBucket
1112
glueDb: glueAlpha.IDatabase
1213
athenaWorkgroup: athena.CfnWorkGroup
1314
}
@@ -18,11 +19,11 @@ export class DdbExport extends Construct {
1819

1920
const getSqlString = (file: string): string => {
2021
let createTableCommand = readFileSync(join(__dirname, `${file}`), 'utf-8').toString()
21-
const s3Location = `s3://${props.bucket.bucketName}/ddb-exports/AWSDynamoDB/ddb-export-id/data/`;
22+
const s3Location = `s3://${props.firehoseBucket.bucketName}/ddb-exports/AWSDynamoDB/ddb-export-id/data/`
2223
createTableCommand = createTableCommand.replace(/s3Location/g, s3Location)
2324
return createTableCommand
2425
}
25-
26+
2627
const queryStringCreateTable = getSqlString('createTable.sql')
2728
const queryStringReadTable = getSqlString('readTable.sql')
2829

@@ -32,7 +33,7 @@ export class DdbExport extends Construct {
3233
environment: {
3334
REGION: Stack.of(this).region,
3435
DYNAMO_DB_TABLE_ARN: props.table.tableArn,
35-
S3_BUCKET_NAME: props.bucket.bucketName,
36+
S3_BUCKET_NAME: props.firehoseBucket.bucketName,
3637
GLUE_DATABASE_NAME: props.glueDb.databaseName,
3738
ATHENA_WORKGROUP_NAME: props.athenaWorkgroup.name,
3839
ATHENA_QUERY_STRING_CREATE_TABLE: queryStringCreateTable,
@@ -44,26 +45,54 @@ export class DdbExport extends Construct {
4445
actions: ['dynamodb:ExportTableToPointInTime'],
4546
resources: [props.table.tableArn],
4647
})
47-
);
48+
)
4849
ddbExportAthenaQuery.addToRolePolicy(
4950
new iam.PolicyStatement({
5051
actions: ['s3:PutObject'],
51-
resources: [`${props.bucket.bucketArn}/*`],
52+
resources: [props.firehoseBucket.bucketArn, `${props.firehoseBucket.bucketArn}/*`,
53+
props.athenaResultBucket.bucketArn, `${props.athenaResultBucket.bucketArn}/*`],
5254
})
5355
);
5456
ddbExportAthenaQuery.addToRolePolicy(
5557
new iam.PolicyStatement({
56-
actions: ['kms:Decrypt'],
57-
resources: [props.bucket.encryptionKey!.keyArn],
58+
actions: ['kms:Decrypt', 'kms:GenerateDataKey'],
59+
resources: [props.firehoseBucket.encryptionKey!.keyArn, props.athenaResultBucket.encryptionKey!.keyArn],
5860
})
59-
);
61+
)
6062
ddbExportAthenaQuery.addToRolePolicy(
6163
new iam.PolicyStatement({
62-
actions: ['athena:CreateNamedQuery', 'athena:ListNamedQueries', 'athena:GetNamedQuery', 'athena:UpdateNamedQuery'],
64+
actions: ['athena:CreateNamedQuery', 'athena:ListNamedQueries', 'athena:GetNamedQuery', 'athena:UpdateNamedQuery', 'athena:StartQueryExecution'],
6365
resources: [`arn:aws:athena:${Stack.of(this).region}:${Stack.of(this).account}:workgroup/${props.athenaWorkgroup.name}`],
6466
})
6567
);
66-
67-
68+
ddbExportAthenaQuery.addToRolePolicy(
69+
new iam.PolicyStatement({
70+
actions: [
71+
'glue:BatchCreatePartition',
72+
'glue:BatchDeletePartition',
73+
'glue:BatchDeleteTable',
74+
'glue:BatchGetPartition',
75+
'glue:CreateDatabase',
76+
'glue:CreatePartition',
77+
'glue:CreateTable',
78+
'glue:DeleteDatabase',
79+
'glue:DeletePartition',
80+
'glue:DeleteTable',
81+
'glue:GetDatabase',
82+
'glue:GetDatabases',
83+
'glue:GetPartition',
84+
'glue:GetPartitions',
85+
'glue:GetTable',
86+
'glue:GetTables',
87+
'glue:UpdateDatabase',
88+
'glue:UpdatePartition',
89+
'glue:UpdateTable'
90+
],
91+
resources: [
92+
`arn:aws:glue:${Stack.of(this).region}:${Stack.of(this).account}:catalog`,
93+
props.glueDb.databaseArn,
94+
`arn:aws:glue:${Stack.of(this).region}:${Stack.of(this).account}:table/${props.glueDb.databaseName}/*`],
95+
})
96+
);
6897
}
6998
}

0 commit comments

Comments
 (0)