Skip to content

Commit 894d001

Browse files
tomcoombs-awstomcoombs-aws
authored
Add carbon exports
1 parent 86c18bd commit 894d001

File tree

1 file changed

+204
-5
lines changed

1 file changed

+204
-5
lines changed

‎data-exports/deploy/data-exports-aggregation.yaml‎

Lines changed: 204 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ Metadata:
1414
- ManageCUR2
1515
- ManageFOCUS
1616
- ManageCOH
17+
- ManageCarbon
1718
-
1819
Label:
1920
default: "Parameters needed in Destination (Data Collection) Account only"
@@ -35,6 +36,8 @@ Metadata:
3536
default: "CUR 2.0"
3637
ManageFOCUS:
3738
default: "FOCUS"
39+
ManageCarbon:
40+
default: "Carbon Emissions"
3841
DestinationAccountId:
3942
default: "Destination (Data Collection) Account Id"
4043
ResourcePrefix:
@@ -81,6 +84,11 @@ Parameters:
8184
Description: "NOTE: you must have Cost Optimization Hub Enabled"
8285
AllowedValues: ['yes', 'no']
8386
Default: "no"
87+
ManageCarbon:
88+
Type: String
89+
Description: ""
90+
AllowedValues: ["yes", "no"]
91+
Default: "no"
8492

8593
##
8694
# Destination specific params
@@ -152,12 +160,14 @@ Conditions:
152160
ManageCUR2: !Equals [!Ref ManageCUR2, 'yes']
153161
ManageFOCUS: !Equals [!Ref ManageFOCUS, 'yes']
154162
ManageCOH: !Equals [!Ref ManageCOH, 'yes']
163+
ManageCarbon: !Equals [!Ref ManageCarbon, "yes"]
155164
EnableSCAD: !Equals [!Ref EnableSCAD, 'yes']
156165
DeployDataExport:
157166
Fn::Or:
158167
- !Condition ManageCUR2
159168
- !Condition ManageFOCUS
160169
- !Condition ManageCOH
170+
- !Condition ManageCarbon
161171
DeploySourceS3:
162172
Fn::And:
163173
- !Condition IsSourceAccount
@@ -166,19 +176,23 @@ Conditions:
166176
DeployCUR2ViaCFN: !And [!Condition IsSourceAccount, !Condition ManageCUR2, !Condition RegionSupportsDataExportsViaCFN]
167177
DeployFOCUSViaCFN: !And [!Condition IsSourceAccount, !Condition ManageFOCUS, !Condition RegionSupportsDataExportsViaCFN]
168178
DeployCOHViaCFN: !And [!Condition IsSourceAccount, !Condition ManageCOH, !Condition RegionSupportsDataExportsViaCFN]
169-
DeployCUR2ViaLambda: !And [!Condition IsSourceAccount, !Condition ManageCUR2, !Not [!Condition RegionSupportsDataExportsViaCFN]]
170-
DeployFOCUSViaLambda: !And [!Condition IsSourceAccount, !Condition ManageFOCUS, !Not [!Condition RegionSupportsDataExportsViaCFN]]
171-
DeployCOHViaLambda: !And [!Condition IsSourceAccount, !Condition ManageCOH, !Not [!Condition RegionSupportsDataExportsViaCFN]]
179+
DeployCarbonViaCFN: !And [!Condition IsSourceAccount, !Condition ManageCarbon, !Condition RegionSupportsDataExportsViaCFN]
180+
DeployCUR2ViaLambda: !And [!Condition IsSourceAccount, !Condition ManageCUR2, !Not [!Condition RegionSupportsDataExportsViaCFN]]
181+
DeployFOCUSViaLambda: !And [!Condition IsSourceAccount, !Condition ManageFOCUS, !Not [!Condition RegionSupportsDataExportsViaCFN]]
182+
DeployCOHViaLambda: !And [!Condition IsSourceAccount, !Condition ManageCOH, !Not [!Condition RegionSupportsDataExportsViaCFN]]
183+
DeployCarbonViaLambda: !And [!Condition IsSourceAccount, !Condition ManageCarbon, !Not [!Condition RegionSupportsDataExportsViaCFN]]
172184
DeployCUR2Table: !And [!Condition IsDestinationAccount, !Condition ManageCUR2]
173185
DeployFOCUSTable: !And [!Condition IsDestinationAccount, !Condition ManageFOCUS]
174186
DeployCOHTable: !And [!Condition IsDestinationAccount, !Condition ManageCOH]
175-
DeployAnyExportViaLambda: !Or [!Condition DeployCUR2ViaLambda, !Condition DeployFOCUSViaLambda, !Condition DeployCOHViaLambda]
176-
DeployAnyTable: !Or [!Condition DeployFOCUSTable, !Condition DeployCUR2Table, !Condition DeployCOHTable]
187+
DeployCarbonTable: !And [!Condition IsDestinationAccount, !Condition ManageCarbon]
188+
DeployAnyExportViaLambda: !Or [!Condition DeployCUR2ViaLambda, !Condition DeployFOCUSViaLambda, !Condition DeployCOHViaLambda, !Condition DeployCarbonViaLambda]
189+
DeployAnyTable: !Or [!Condition DeployFOCUSTable, !Condition DeployCUR2Table, !Condition DeployCOHTable, !Condition DeployCarbonTable]
177190

178191
NeedLakeFormationEnabledDB: !And [ !Equals [ !Ref LakeFormationEnabled, "yes"], !Condition DeployAnyTable]
179192
NeedLakeFormationEnabledCOH: !And [!Condition NeedLakeFormationEnabledDB, !Condition ManageCOH]
180193
NeedLakeFormationEnabledCUR2: !And [!Condition NeedLakeFormationEnabledDB, !Condition ManageCUR2]
181194
NeedLakeFormationEnabledFOCUS: !And [!Condition NeedLakeFormationEnabledDB, !Condition ManageFOCUS]
195+
NeedLakeFormationEnabledCarbon: !And [!Condition NeedLakeFormationEnabledDB, !Condition ManageCarbon]
182196
DeployWriteBlocker:
183197
Fn::And:
184198
- !Condition IsDestinationAccount
@@ -202,6 +216,10 @@ Mappings:
202216
DefaultQuery: >-
203217
SELECT account_id, action_type, currency_code, current_resource_details, current_resource_summary, current_resource_type, estimated_monthly_cost_after_discount, estimated_monthly_cost_before_discount, estimated_monthly_savings_after_discount, estimated_monthly_savings_before_discount, estimated_savings_percentage_after_discount, estimated_savings_percentage_before_discount, implementation_effort, last_refresh_timestamp, recommendation_id, recommendation_lookback_period_in_days, recommendation_source, recommended_resource_details, recommended_resource_summary, recommended_resource_type, region, resource_arn, restart_needed, rollback_possible, tags
204218
FROM COST_OPTIMIZATION_RECOMMENDATIONS
219+
Carbon:
220+
DefaultQuery: >-
221+
SELECT last_refresh_timestamp, location, model_version, payer_account_id, product_code, region_code, total_mbm_emissions_unit, total_mbm_emissions_value, usage_account_id, usage_period_end, usage_period_start
222+
FROM CARBON_EMISSIONS
205223
206224
Resources:
207225

@@ -379,6 +397,16 @@ Resources:
379397
Priority: 3
380398
DeleteMarkerReplication:
381399
Status: Enabled
400+
- Destination:
401+
Bucket: !Sub "arn:${AWS::Partition}:s3:::${ResourcePrefix}-${DestinationAccountId}-data-exports"
402+
StorageClass: STANDARD
403+
Id: ReplicateCarbonData
404+
Filter:
405+
Prefix: !Sub "carbon/${AWS::AccountId}/${ResourcePrefix}-carbon/data/" # Hardcoded export name
406+
Status: Enabled
407+
Priority: 4
408+
DeleteMarkerReplication:
409+
Status: Enabled
382410
- Fn::If:
383411
- NonEmptySecondaryDestinationBucket
384412
- Destination:
@@ -418,6 +446,19 @@ Resources:
418446
DeleteMarkerReplication:
419447
Status: Enabled
420448
- !Ref 'AWS::NoValue'
449+
- Fn::If:
450+
- NonEmptySecondaryDestinationBucket
451+
- Destination:
452+
Bucket: !Sub "arn:${AWS::Partition}:s3:::${SecondaryDestinationBucket}"
453+
StorageClass: STANDARD
454+
Id: ReplicateCarbonDataToSecondaryBucket
455+
Filter:
456+
Prefix: !Sub "carbon/${AWS::AccountId}/${ResourcePrefix}-carbon/data/" # Hardcoded export name
457+
Status: Enabled
458+
Priority: 104
459+
DeleteMarkerReplication:
460+
Status: Enabled
461+
- !Ref "AWS::NoValue"
421462
LifecycleConfiguration:
422463
Rules:
423464
- Id: Object&Version Expiration
@@ -978,6 +1019,58 @@ Resources:
9781019
ServiceToken: !GetAtt CreateServiceLinkedRoleFunction.Arn
9791020
ServiceTimeout: "90"
9801021

1022+
# Carbon Emissions
1023+
1024+
## Deploy Data Export natively via CFN resource in regions that support native CFN
1025+
LocalCarbonViaCFN:
1026+
Type: AWS::BCMDataExports::Export
1027+
Condition: DeployCarbonViaCFN
1028+
DependsOn:
1029+
- SourceS3BucketPolicy
1030+
Properties:
1031+
Export:
1032+
DataQuery:
1033+
QueryStatement: !FindInMap [DataExports, Carbon, DefaultQuery]
1034+
Description: "Carbon emissions export for aggregation in CID"
1035+
DestinationConfigurations:
1036+
S3Destination:
1037+
S3Bucket: !Ref SourceS3
1038+
S3Prefix: !Sub "carbon/${AWS::AccountId}" # Explicitly hardcode name of folder (it will be the same as table name)
1039+
S3Region: !Ref AWS::Region
1040+
S3OutputConfigurations:
1041+
Overwrite: "OVERWRITE_REPORT"
1042+
Format: "PARQUET"
1043+
Compression: "PARQUET"
1044+
OutputType: "CUSTOM"
1045+
Name: !Sub '${ResourcePrefix}-carbon'
1046+
RefreshCadence:
1047+
Frequency: "SYNCHRONOUS"
1048+
1049+
# Deploy Data Export via Lambda due to missing CFN resource definition AWS::BCMDataExports::Export outside us-east-1 or cn-northwest-1
1050+
LocalCarbonViaCustomResource:
1051+
Type: Custom::CUR2Creator
1052+
Condition: DeployCarbonViaLambda
1053+
Properties:
1054+
ServiceToken: !GetAtt CidDataExportCreatorLambda.Arn
1055+
BucketPolicyWait: !Ref SourceS3BucketPolicy
1056+
Export:
1057+
DataQuery:
1058+
QueryStatement: !FindInMap [DataExports, Carbon, DefaultQuery]
1059+
Description: "Carbon emissions export for aggregation in CID"
1060+
DestinationConfigurations:
1061+
S3Destination:
1062+
S3Bucket: !Ref SourceS3
1063+
S3Prefix: !Sub "carbon/${AWS::AccountId}" # Explicitly hardcode name of folder (it will be the same as table name)
1064+
S3Region: !Ref AWS::Region
1065+
S3OutputConfigurations:
1066+
Overwrite: "OVERWRITE_REPORT"
1067+
Format: "PARQUET"
1068+
Compression: "PARQUET"
1069+
OutputType: "CUSTOM"
1070+
Name: !Sub '${ResourcePrefix}-carbon'
1071+
RefreshCadence:
1072+
Frequency: "SYNCHRONOUS"
1073+
9811074
###########################################################################
9821075
# Lambda DataExport Creator: used to create DataExport from outside us-east-1 or cn-northwest-1
9831076
###########################################################################
@@ -1033,6 +1126,9 @@ Resources:
10331126
- cost-optimization-hub:GetRecommendation #need this permission for bcm-data-exports to work
10341127
- cost-optimization-hub:ListRecommendations
10351128
Resource: "arn:aws:cost-optimization-hub:*"
1129+
- Effect: Allow
1130+
Action: sustainability:GetCarbonFootprintSummary # needed ad carbon emissions data export creation time
1131+
Resource: "*"
10361132

10371133
CidDataExportCreatorLambda:
10381134
Type: AWS::Lambda::Function
@@ -1500,6 +1596,95 @@ Resources:
15001596
}
15011597
}
15021598
1599+
###########################################################################
1600+
# Carbon emissions
1601+
###########################################################################
1602+
1603+
CarbonTable: # Initial creation of table. it will be updated by crawler later
1604+
Type: AWS::Glue::Table
1605+
Condition: DeployCarbonTable
1606+
Properties:
1607+
CatalogId: !Sub "${AWS::AccountId}"
1608+
DatabaseName: !Ref CIDDatabase
1609+
TableInput:
1610+
Name: carbon
1611+
Owner: owner
1612+
Retention: 0
1613+
TableType: EXTERNAL_TABLE
1614+
Parameters:
1615+
compressionType: none
1616+
classification: parquet
1617+
UPDATED_BY_CRAWLER: !Ref CarbonCrawler
1618+
StorageDescriptor:
1619+
BucketColumns: []
1620+
Compressed: false
1621+
Location: !Sub "s3://${DestinationS3}/carbon/"
1622+
NumberOfBuckets: -1
1623+
InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat
1624+
OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat
1625+
SerdeInfo:
1626+
Parameters:
1627+
serialization.format: '1'
1628+
SerializationLibrary: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
1629+
StoredAsSubDirectories: false
1630+
Columns: # All fields required for CID
1631+
- {"Name": "payer_account_id", "Type": "string"}
1632+
- {"Name": "usage_account_id", "Type": "string"}
1633+
- {"Name": "total_mbm_emissions_value", "Type": "double"}
1634+
- {"Name": "total_mbm_emissions_unit", "Type": "string"}
1635+
- {"Name": "model_version", "Type": "string"}
1636+
- {"Name": "product_code", "Type": "string"}
1637+
- {"Name": "usage_period_start", "Type": "timestamp"}
1638+
- {"Name": "usage_period_end", "Type": "timestamp"}
1639+
- {"Name": "last_refresh_timestamp", "Type": "timestamp"}
1640+
- {"Name": "region_code", "Type": "string"}
1641+
- {"Name": "location", "Type": "string"}
1642+
PartitionKeys:
1643+
- {"Name": "source_account_id", "Type": "string" }
1644+
- {"Name": "report_name", "Type": "string" }
1645+
- {"Name": "data", "Type": "string" }
1646+
- {"Name": "carbon_model_version", "Type": "string" }
1647+
- {"Name": "usage_period", "Type": "string" }
1648+
1649+
CarbonCrawler:
1650+
Type: AWS::Glue::Crawler
1651+
Condition: DeployCarbonTable
1652+
Properties:
1653+
Name: !Sub "${ResourcePrefix}-DataExportCarbonCrawler"
1654+
Description: A recurring crawler that keeps your Carbon emissions table in Athena up-to-date.
1655+
Role: !GetAtt CidDataExportCrawlerRole.Arn
1656+
DatabaseName: !Ref CIDDatabase
1657+
Targets:
1658+
S3Targets:
1659+
- Path: !Sub "s3://${DestinationS3}/carbon/"
1660+
Exclusions:
1661+
- '**.json'
1662+
- '**.yml'
1663+
- '**.sql'
1664+
- '**.csv'
1665+
- '**.csv.metadata'
1666+
- '**.gz'
1667+
- '**.zip'
1668+
- '**/cost_and_usage_data_status/*'
1669+
- 'aws-programmatic-access-test-object'
1670+
SchemaChangePolicy:
1671+
DeleteBehavior: LOG
1672+
RecrawlPolicy:
1673+
RecrawlBehavior: CRAWL_EVERYTHING
1674+
Schedule:
1675+
ScheduleExpression: cron(0 2 * * ? *)
1676+
Configuration: |
1677+
{
1678+
"Version":1.0,
1679+
"Grouping": {
1680+
"TableGroupingPolicy": "CombineCompatibleSchemas"
1681+
},
1682+
"CrawlerOutput":{
1683+
"Tables":{
1684+
"AddOrUpdateBehavior":"MergeNewColumns"
1685+
}
1686+
}
1687+
}
15031688
15041689
LakeFormationTagsForDatabase:
15051690
Type: AWS::LakeFormation::TagAssociation
@@ -1560,6 +1745,20 @@ Resources:
15601745
- !ImportValue cid-LakeFormation-TagValue
15611746
CatalogId: !Ref "AWS::AccountId"
15621747

1748+
LakeFormationTagsForCarbon:
1749+
Type: AWS::LakeFormation::TagAssociation
1750+
Condition: NeedLakeFormationEnabledCarbon
1751+
Properties:
1752+
Resource:
1753+
Table:
1754+
CatalogId: !Ref "AWS::AccountId"
1755+
DatabaseName: !Ref CIDDatabase
1756+
Name: !Ref CarbonTable
1757+
LFTags:
1758+
- TagKey: !ImportValue cid-LakeFormation-TagKey # you need to install cid-lakeformation-prerequisite.yaml
1759+
TagValues:
1760+
- !ImportValue cid-LakeFormation-TagValue
1761+
CatalogId: !Ref "AWS::AccountId"
15631762

15641763
###########################################################################
15651764
# Generic Resources

0 commit comments

Comments
 (0)