Merge pull request #181 from awslabs/feat/db-cluster-inventory-collection

Inventory collection for RDS DB Clusters

Author: juchavw

data-collection/README.md

@@ -26,7 +26,7 @@ List of modules and objects collected:
 | `trusted-advisor`            | AWS Trusted Advisor   | Linked Account      | Requires Enterpriso or OnRamp Support Level |
 | `cost-explorer-cost-anomaly` | AWS Anomalies         | Management Account  |          |
 | `cost-explorer-rightsizing`  | AWS Cost Explorer     | Management Account  | DEPRECATED. Please use `cost-optimization-hub` |
-| `inventory`                  | Various services      | Linked Account      | Collects `Amazon OpenSearch Domains`, `Amazon ElastiCache Clusters`, `RDS DB Instances`, `EBS Volumes`, `AMI`, `EC2 Instances`, `EBS Snapshot`, `RDS Snapshot`, `Lambda` |
+| `inventory`                  | Various services      | Linked Account      | Collects `Amazon OpenSearch Domains`, `Amazon ElastiCache Clusters`, `RDS DB Instances`, `EBS Volumes`, `AMI`, `EC2 Instances`, `EBS Snapshot`, `RDS Snapshot`, `Lambda`, `RDS DB Clusters`, `EKS Clusters` |
 | `pricing`                    | Various services      | N/A                 | Collects pricing for `Amazon RDS`, `Amazon EC2`, `Amazon ElastiCache`, `AWS Lambda`, `Amazon OpenSearch`, `AWS Compute Savings Plan` |
 | `rds-usage`                  |  Amazon RDS           | Linked Account      | Collects CloudWatch metrics for chargeback |
 | `transit-gateway`            |  AWS Transit Gateway  | Linked Account      | Collects CloudWatch metrics for chargeback |

data-collection/deploy/deploy-in-linked-account.yaml

@@ -184,6 +184,7 @@ Resources:
               - "ec2:DescribeRegions"
               - "ec2:DescribeInstances"
               - "ec2:DescribeVpcs"
+              - "rds:DescribeDBClusters"
               - "rds:DescribeDBInstances"
               - "rds:DescribeDBSnapshots"
               - "es:ListDomainNames"

data-collection/deploy/module-inventory.yaml

@@ -56,7 +56,7 @@ Parameters:
     Description: ARN of a Lambda for Managing GlueTable
   AwsObjects:
     Type: CommaDelimitedList
-    Default: OpensearchDomains, ElasticacheClusters, RdsDbInstances, EBS, AMI, Snapshot, Ec2Instances, VpcInstances, RdsDbSnapshots, EKSClusters, LambdaFunctions
+    Default: OpensearchDomains, ElasticacheClusters, RdsDbInstances, EBS, AMI, Snapshot, Ec2Instances, VpcInstances, RdsDbSnapshots, EKSClusters, LambdaFunctions, RdsDbClusters
     Description: Services for pulling price data
 
 Mappings:
@@ -189,6 +189,141 @@ Mappings:
               SerializationLibrary: org.openx.data.jsonserde.JsonSerDe
           TableType: EXTERNAL_TABLE
 
+    RdsDbClusters:
+      path: rds-db-clusters
+      table:
+        - Name: inventory_rds_db_clusters_data
+          TableType: EXTERNAL_TABLE
+
+          PartitionKeys:
+          - { Name: payer_id, Type: string }
+          - { Name: year, Type: string }
+          - { Name: month, Type: string }
+          - { Name: day, Type: string }
+          StorageDescriptor:
+            Columns:
+            - Name: readreplicaidentifiers
+              Type: string
+            - Name: capacity
+              Type: string
+            - Name: charactersetname
+              Type: string
+            - Name: preferredbackupwindow
+              Type: string
+            - Name: copytagstosnapshot
+              Type: string
+            - Name: activitystreamkinesisstreamname
+              Type: string
+            - Name: multiaz
+              Type: string
+            - Name: backtrackwindow
+              Type: string
+            - Name: earliestrestorabletime
+              Type: string
+            - Name: engine
+              Type: string
+            - Name: earliestbacktracktime
+              Type: string
+            - Name: httpendpointenabled
+              Type: string
+            - Name: percentprogress
+              Type: string
+            - Name: dbclusterparametergroup
+              Type: string
+            - Name: hostedzoneid
+              Type: string
+            - Name: status
+              Type: string
+            - Name: associatedroles
+              Type: string
+            - Name: dbclusteridentifier
+              Type: string
+            - Name: backtrackconsumedchangerecords
+              Type: string
+            - Name: iamdatabaseauthenticationenabled
+              Type: string
+            - Name: serverlessv2scalingconfiguration
+              Type: string
+            - Name: customendpoints
+              Type: string
+            - Name: port
+              Type: string
+            - Name: activitystreamkmskeyid
+              Type: string
+            - Name: masterusername
+              Type: string
+            - Name: dbclusterresourceid
+              Type: string
+            - Name: clustercreatetime
+              Type: string
+            - Name: vpcsecuritygroups
+              Type: string
+            - Name: dbsubnetgroup
+              Type: string
+            - Name: latestrestorabletime
+              Type: string
+            - Name: availabilityzones
+              Type: string
+            - Name: dbclusteroptiongroupmemberships
+              Type: string
+            - Name: replicationsourceidentifier
+              Type: string
+            - Name: deletionprotection
+              Type: string
+            - Name: endpoint
+              Type: string
+            - Name: preferredmaintenancewindow
+              Type: string
+            - Name: readerendpoint
+              Type: string
+            - Name: accountid
+              Type: string
+            - Name: enabledcloudwatchlogsexports
+              Type: string
+            - Name: scalingconfigurationinfo
+              Type: string
+            - Name: storageencrypted
+              Type: string
+            - Name: enginemode
+              Type: string
+            - Name: databasename
+              Type: string
+            - Name: activitystreammode
+              Type: string
+            - Name: crossaccountclone
+              Type: string
+            - Name: dbclusterarn
+              Type: string
+            - Name: activitystreamstatus
+              Type: string
+            - Name: allocatedstorage
+              Type: string
+            - Name: engineversion
+              Type: string
+            - Name: backupretentionperiod
+              Type: string
+            - Name: clonegroupid
+              Type: string
+            - Name: dbclustermembers
+              Type: string
+            - Name: kmskeyid
+              Type: string
+            - Name: accountid
+              Type: string
+            - Name: collection_date
+              Type: string
+            - Name: region
+              Type: string
+            InputFormat: org.apache.hadoop.mapred.TextInputFormat
+            Location: !Sub s3://${DestinationBucket}/inventory/inventory-rds-db-clusters-data/
+            OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+            Parameters:
+              UPDATED_BY_CRAWLER: !Sub ${ResourcePrefix}inventory-RdsDbClusters-Crawler
+            SerdeInfo:
+              Parameters:
+                paths: readreplicaidentifiers,capacity,charactersetname,preferredbackupwindow,copytagstosnapshot,activitystreamkinesisstreamname,multiaz,backtrackwindow,earliestrestorabletime,engine,earliestbacktracktime,httpendpointenabled,payer_id,percentprogress,day,dbclusterparametergroup,hostedzoneid,status,associatedroles,dbclusteridentifier,backtrackconsumedchangerecords,iamdatabaseauthenticationenabled,serverlessv2scalingconfiguration,customendpoints,port,activitystreamkmskeyid,masterusername,dbclusterresourceid,clustercreatetime,vpcsecuritygroups,dbsubnetgroup,latestrestorabletime,availabilityzones,dbclusteroptiongroupmemberships,replicationsourceidentifier,year,deletionprotection,endpoint,preferredmaintenancewindow,readerendpoint,accountid,enabledcloudwatchlogsexports,scalingconfigurationinfo,storageencrypted,enginemode,databasename,activitystreammode,month,crossaccountclone,dbclusterarn,activitystreamstatus,allocatedstorage,engineversion,backupretentionperiod,clonegroupid,dbclustermembers,kmskeyid,accountid,collection_date,region
+              SerializationLibrary: org.openx.data.jsonserde.JsonSerDe
+
     RdsDbInstances:
       path: rds-db-instances
       table:
@@ -1043,6 +1178,12 @@ Resources:
                       obj_name='CacheClusters'
                       # fields=['CacheClusterId', 'CacheNodeType', 'EngineVersion', 'Engine', 'NumCacheNodes', 'PreferredAvailabilityZone', 'CacheClusterCreateTime'],
                   ),
+                  'rds-db-clusters': partial(
+                      paginated_scan,
+                      service='rds',
+                      function_name='describe_db_clusters',
+                      obj_name='DBClusters[*]'
+                  ),
                   'rds-db-instances': partial(
                       paginated_scan,
                       service='rds',