Skip to content

Commit 7cf1177

Browse files
dlpzxdlpzx
authored
Merge pull request #71 from awslabs/34-quicksight-dashboard-connected-to-rds-metadata-database-for-platform-monitoring
quicksight dashboard connected to rds metadata database for platform monitoring
2 parents 7eabba7 + a0a1658 commit 7cf1177

30 files changed

+1083
-10
lines changed

‎backend/dataall/api/Objects/Tenant/mutations.py‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
from ... import gql
22
from .input_types import UpdateGroupTenantPermissionsInput
3-
from .resolvers import update_group_permissions
3+
from .resolvers import *
44

55
updateGroupPermission = gql.MutationField(
66
name='updateGroupTenantPermissions',
@@ -12,3 +12,22 @@
1212
type=gql.Boolean,
1313
resolver=update_group_permissions,
1414
)
15+
16+
createQuicksightDataSourceSet = gql.MutationField(
17+
name='createQuicksightDataSourceSet',
18+
args=[
19+
gql.Argument(name='vpcConnectionId', type=gql.NonNullableType(gql.String))
20+
],
21+
type=gql.String,
22+
resolver=create_quicksight_data_source_set,
23+
)
24+
25+
updateSSMParameter = gql.MutationField(
26+
name='updateSSMParameter',
27+
args=[
28+
gql.Argument(name='name', type=gql.NonNullableType(gql.String)),
29+
gql.Argument(name='value', type=gql.NonNullableType(gql.String))
30+
],
31+
type=gql.String,
32+
resolver=update_ssm_parameter,
33+
)

‎backend/dataall/api/Objects/Tenant/queries.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,33 @@
1616
type=gql.Ref('GroupSearchResult'),
1717
resolver=list_tenant_groups,
1818
)
19+
20+
getMonitoringDashboardId = gql.QueryField(
21+
name='getMonitoringDashboardId',
22+
type=gql.String,
23+
resolver=get_monitoring_dashboard_id,
24+
)
25+
26+
getMonitoringVpcConnectionId = gql.QueryField(
27+
name='getMonitoringVPCConnectionId',
28+
type=gql.String,
29+
resolver=get_monitoring_vpc_connection_id,
30+
)
31+
32+
getPlatformAuthorSession = gql.QueryField(
33+
name='getPlatformAuthorSession',
34+
args=[
35+
gql.Argument(name='awsAccount', type=gql.NonNullableType(gql.String)),
36+
],
37+
type=gql.String,
38+
resolver=get_quicksight_author_session,
39+
)
40+
41+
getPlatformReaderSession = gql.QueryField(
42+
name='getPlatformReaderSession',
43+
args=[
44+
gql.Argument(name='dashboardId', type=gql.NonNullableType(gql.String)),
45+
],
46+
type=gql.String,
47+
resolver=get_quicksight_reader_session,
48+
)

‎backend/dataall/api/Objects/Tenant/resolvers.py‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,10 @@
1+
import os
2+
13
from .... import db
4+
from ....aws.handlers.sts import SessionHelper
5+
from ....aws.handlers.parameter_store import ParameterStoreManager
6+
from ....aws.handlers.quicksight import Quicksight
7+
from ....db import exceptions
28

39

410
def update_group_permissions(context, source, input=None):
@@ -32,3 +38,95 @@ def list_tenant_groups(context, source, filter=None):
3238
data=filter,
3339
check_perm=True,
3440
)
41+
42+
43+
def update_ssm_parameter(context, source, name: str = None, value: str = None):
44+
current_account = SessionHelper.get_account()
45+
region = os.getenv('AWS_REGION', 'eu-west-1')
46+
print(value)
47+
print(name)
48+
response = ParameterStoreManager.update_parameter(AwsAccountId=current_account, region=region, parameter_name=f'/dataall/{os.getenv("envname", "local")}/quicksightmonitoring/{name}', parameter_value=value)
49+
return response
50+
51+
52+
def get_monitoring_dashboard_id(context, source):
53+
current_account = SessionHelper.get_account()
54+
region = os.getenv('AWS_REGION', 'eu-west-1')
55+
dashboard_id = ParameterStoreManager.get_parameter_value(AwsAccountId=current_account, region=region, parameter_path=f'/dataall/{os.getenv("envname", "local")}/quicksightmonitoring/DashboardId')
56+
if not dashboard_id:
57+
raise exceptions.AWSResourceNotFound(
58+
action='GET_DASHBOARD_ID',
59+
message='Dashboard Id could not be found on AWS Parameter Store',
60+
)
61+
return dashboard_id
62+
63+
64+
def get_monitoring_vpc_connection_id(context, source):
65+
current_account = SessionHelper.get_account()
66+
region = os.getenv('AWS_REGION', 'eu-west-1')
67+
vpc_connection_id = ParameterStoreManager.get_parameter_value(AwsAccountId=current_account, region=region, parameter_path=f'/dataall/{os.getenv("envname", "local")}/quicksightmonitoring/VPCConnectionId')
68+
if not vpc_connection_id:
69+
raise exceptions.AWSResourceNotFound(
70+
action='GET_VPC_CONNECTION_ID',
71+
message='Dashboard Id could not be found on AWS Parameter Store',
72+
)
73+
return vpc_connection_id
74+
75+
76+
def create_quicksight_data_source_set(context, source, vpcConnectionId: str = None):
77+
current_account = SessionHelper.get_account()
78+
region = os.getenv('AWS_REGION', 'eu-west-1')
79+
user = Quicksight.register_user(AwsAccountId=current_account, UserName=context.username, UserRole='AUTHOR')
80+
81+
datasourceId = Quicksight.create_data_source_vpc(AwsAccountId=current_account, region=region, UserName=context.username, vpcConnectionId=vpcConnectionId)
82+
# Data sets are not created programmatically. Too much overhead for the value added. However, an example API is provided:
83+
# datasets = Quicksight.create_data_set_from_source(AwsAccountId=current_account, region=region, UserName='dataallTenantUser', dataSourceId=datasourceId, tablesToImport=['organization', 'environment', 'dataset', 'datapipeline', 'dashboard', 'share_object'])
84+
85+
return datasourceId
86+
87+
88+
def get_quicksight_author_session(context, source, awsAccount: str = None):
89+
with context.engine.scoped_session() as session:
90+
admin = db.api.TenantPolicy.is_tenant_admin(context.groups)
91+
92+
if not admin:
93+
raise db.exceptions.TenantUnauthorized(
94+
username=context.username,
95+
action=db.permissions.TENANT_ALL,
96+
tenant_name=context.username,
97+
)
98+
region = os.getenv('AWS_REGION', 'eu-west-1')
99+
100+
url = Quicksight.get_author_session(
101+
AwsAccountId=awsAccount,
102+
region=region,
103+
UserName=context.username,
104+
UserRole='AUTHOR',
105+
)
106+
107+
return url
108+
109+
110+
def get_quicksight_reader_session(context, source, dashboardId: str = None):
111+
with context.engine.scoped_session() as session:
112+
admin = db.api.TenantPolicy.is_tenant_admin(context.groups)
113+
114+
if not admin:
115+
raise db.exceptions.TenantUnauthorized(
116+
username=context.username,
117+
action=db.permissions.TENANT_ALL,
118+
tenant_name=context.username,
119+
)
120+
121+
region = os.getenv('AWS_REGION', 'eu-west-1')
122+
current_account = SessionHelper.get_account()
123+
124+
url = Quicksight.get_reader_session(
125+
AwsAccountId=current_account,
126+
region=region,
127+
UserName=context.username,
128+
UserRole='READER',
129+
DashboardId=dashboardId
130+
)
131+
132+
return url

‎backend/dataall/aws/handlers/parameter_store.py‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,26 @@ def client(AwsAccountId, region):
2323
@staticmethod
2424
def get_parameter_value(AwsAccountId, region, parameter_path):
2525
if not parameter_path:
26-
raise Exception('Secret name is None')
26+
raise Exception('Parameter name is None')
2727
try:
2828
parameter_value = ParameterStoreManager.client(
2929
AwsAccountId, region
3030
).get_parameter(Name=parameter_path)['Parameter']['Value']
3131
except ClientError as e:
3232
raise Exception(e)
3333
return parameter_value
34+
35+
@staticmethod
36+
def update_parameter(AwsAccountId, region, parameter_name, parameter_value):
37+
if not parameter_name:
38+
raise Exception('Parameter name is None')
39+
if not parameter_value:
40+
raise Exception('Parameter value is None')
41+
try:
42+
response = ParameterStoreManager.client(
43+
AwsAccountId, region
44+
).put_parameter(Name=parameter_name, Value=parameter_value, Overwrite=True)['Version']
45+
except ClientError as e:
46+
raise Exception(e)
47+
else:
48+
return str(response)

‎backend/dataall/aws/handlers/quicksight.py‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,13 @@
11
import logging
22
import re
3+
import os
4+
import ast
35

46
from botocore.exceptions import ClientError
57

68
from .sts import SessionHelper
9+
from .secrets_manager import SecretsManager
10+
from .parameter_store import ParameterStoreManager
711

812
logger = logging.getLogger('QuicksightHandler')
913
logger.setLevel(logging.DEBUG)
@@ -262,3 +266,154 @@ def can_import_dashboard(AwsAccountId, region, UserName, DashboardId):
262266
return True
263267

264268
return False
269+
270+
@staticmethod
271+
def create_data_source_vpc(AwsAccountId, region, UserName, vpcConnectionId):
272+
client = Quicksight.get_quicksight_client(AwsAccountId, region)
273+
identity_region = 'us-east-1'
274+
275+
user = Quicksight.register_user(AwsAccountId, UserName, UserRole='AUTHOR')
276+
try:
277+
response = client.describe_data_source(
278+
AwsAccountId=AwsAccountId, DataSourceId="dataall-metadata-db"
279+
)
280+
281+
except client.exceptions.ResourceNotFoundException:
282+
aurora_secret_arn = ParameterStoreManager.get_parameter_value(AwsAccountId=AwsAccountId, region=region, parameter_path=f'/dataall/{os.getenv("envname", "local")}/aurora/secret_arn')
283+
aurora_params = SecretsManager.get_secret_value(
284+
AwsAccountId=AwsAccountId, region=region, secretId=aurora_secret_arn
285+
)
286+
aurora_params_dict = ast.literal_eval(aurora_params)
287+
response = client.create_data_source(
288+
AwsAccountId=AwsAccountId,
289+
DataSourceId="dataall-metadata-db",
290+
Name="dataall-metadata-db",
291+
Type="AURORA_POSTGRESQL",
292+
DataSourceParameters={
293+
'AuroraPostgreSqlParameters': {
294+
'Host': aurora_params_dict["host"],
295+
'Port': aurora_params_dict["port"],
296+
'Database': aurora_params_dict["dbname"]
297+
}
298+
},
299+
Credentials={
300+
"CredentialPair": {
301+
"Username": aurora_params_dict["username"],
302+
"Password": aurora_params_dict["password"],
303+
}
304+
},
305+
Permissions=[
306+
{
307+
"Principal": f"arn:aws:quicksight:{region}:{AwsAccountId}:group/default/dataall",
308+
"Actions": [
309+
"quicksight:UpdateDataSourcePermissions",
310+
"quicksight:DescribeDataSource",
311+
"quicksight:DescribeDataSourcePermissions",
312+
"quicksight:PassDataSource",
313+
"quicksight:UpdateDataSource",
314+
"quicksight:DeleteDataSource"
315+
]
316+
}
317+
],
318+
VpcConnectionProperties={
319+
'VpcConnectionArn': f"arn:aws:quicksight:{region}:{AwsAccountId}:vpcConnection/{vpcConnectionId}"
320+
}
321+
)
322+
323+
return "dataall-metadata-db"
324+
325+
@staticmethod
326+
def create_data_set_from_source(AwsAccountId, region, UserName, dataSourceId, tablesToImport):
327+
client = Quicksight.get_quicksight_client(AwsAccountId, region)
328+
user = Quicksight.describe_user(AwsAccountId, UserName)
329+
if not user:
330+
return False
331+
332+
data_source = client.describe_data_source(
333+
AwsAccountId=AwsAccountId,
334+
DataSourceId=dataSourceId
335+
)
336+
337+
if not data_source:
338+
return False
339+
340+
for table in tablesToImport:
341+
342+
response = client.create_data_set(
343+
AwsAccountId=AwsAccountId,
344+
DataSetId=f"dataall-imported-{table}",
345+
Name=f"dataall-imported-{table}",
346+
PhysicalTableMap={
347+
'string': {
348+
'RelationalTable': {
349+
'DataSourceArn': data_source.get('DataSource').get('Arn'),
350+
'Catalog': 'string',
351+
'Schema': 'dev',
352+
'Name': table,
353+
'InputColumns': [
354+
{
355+
'Name': 'string',
356+
'Type': 'STRING'
357+
},
358+
]
359+
}
360+
}},
361+
ImportMode='DIRECT_QUERY',
362+
Permissions=[
363+
{
364+
'Principal': user.get('Arn'),
365+
'Actions': [
366+
"quicksight:DescribeDataSet",
367+
"quicksight:DescribeDataSetPermissions",
368+
"quicksight:PassDataSet",
369+
"quicksight:DescribeIngestion",
370+
"quicksight:ListIngestions"
371+
]
372+
},
373+
],
374+
)
375+
376+
return True
377+
378+
@staticmethod
379+
def create_analysis(AwsAccountId, region, UserName):
380+
client = Quicksight.get_quicksight_client(AwsAccountId, region)
381+
user = Quicksight.describe_user(AwsAccountId, UserName)
382+
if not user:
383+
return False
384+
385+
response = client.create_analysis(
386+
AwsAccountId=AwsAccountId,
387+
AnalysisId='dataallMonitoringAnalysis',
388+
Name='dataallMonitoringAnalysis',
389+
Permissions=[
390+
{
391+
'Principal': user.get('Arn'),
392+
'Actions': [
393+
'quicksight:DescribeAnalysis',
394+
'quicksight:DescribeAnalysisPermissions',
395+
'quicksight:UpdateAnalysisPermissions',
396+
'quicksight:UpdateAnalysis'
397+
]
398+
},
399+
],
400+
SourceEntity={
401+
'SourceTemplate': {
402+
'DataSetReferences': [
403+
{
404+
'DataSetPlaceholder': 'environment',
405+
'DataSetArn': f"arn:aws:quicksight:{region}:{AwsAccountId}:dataset/<DATASET-ID>"
406+
},
407+
],
408+
'Arn': '<TEMPLATE-THAT-WE-WANT-TO-MIGRATE'
409+
}
410+
},
411+
Tags=[
412+
{
413+
'Key': 'application',
414+
'Value': 'dataall'
415+
},
416+
]
417+
)
418+
419+
return True

0 commit comments

Comments
 (0)