feat: Fix RDS multitenant S3 locations and table structure

- Update Athena tables to match actual S3 data partitioning structure
- Add payer_id and region partition keys to support multi-account data
- Configure Parquet SerDe for proper data format handling
- Remove predefined Glue tables, let crawlers auto-discover schema
- Add hourly_ prefix to separate crawler table creation
- Update pi_data_view to reference hourly_rds_multitenant table
- Update deployment templates to support RDS multitenant module

Author: davidecoccia

data-collection/deploy/deploy-data-collection.yaml

@@ -347,6 +347,7 @@ Conditions:
       - !Condition DeployServiceQuotasModule
       - !Condition DeployEUCUtilizationModule 
       - !Condition DeployComputeOptimizerModule
+      - !Condition DeployRdsMultitenantModule
   RegionsInScopeIsEmpty: !Equals
     - !Join [ '', !Split [ ' ', !Ref RegionsInScope  ] ] # remove spaces
     - ""
@@ -1540,7 +1541,7 @@ Resources:
     Type: AWS::CloudFormation::Stack
     Condition: DeployRdsMultitenantModule
     Properties:
-      TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/v3.12.0/module-rds-multitenant.yaml"
+      TemplateURL: "https://dcoccia-test-static-website.s3.eu-central-1.amazonaws.com/module-rds-multitenant.yaml" 
       Parameters:
         DatabaseName: !Ref DatabaseName
         DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns
@@ -1550,8 +1551,10 @@ Resources:
         GlueRoleARN: !GetAtt GlueRole.Arn
         ResourcePrefix: !Ref ResourcePrefix
         LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn
+        MultiAccountRoleName: !Sub "${ResourcePrefix}${MultiAccountRoleName}"
+        AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}"
         CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ]
-        StepFunctionTemplate: !FindInMap [StepFunctionCode, standalone-state-machine, TemplatePath]
+        StepFunctionTemplate: !FindInMap [StepFunctionCode, main-state-machine, TemplatePath]
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
         RegionsInScope:

data-collection/deploy/deploy-data-read-permissions.yaml

@@ -33,6 +33,7 @@ Metadata:
           - IncludeLicenseManagerModule
           - IncludeServiceQuotasModule
           - IncludeResilienceHubModule
+          - IncludeRdsMultitenantModule
     ParameterLabels:
       ManagementAccountRole:
         default: "Management account role"
@@ -191,6 +192,11 @@ Parameters:
     Description: Collects Resilience Hub information
     AllowedValues: ['yes', 'no']
     Default: 'no'
+  IncludeRdsMultitenantModule:
+    Type: String
+    Description: Collects RDS Performance Insights data for multi-tenant cost allocation
+    AllowedValues: ['yes', 'no']
+    Default: 'no'
 Conditions:
   DeployModuleReadInMgmt: !Equals [!Ref AllowModuleReadInMgmt, "yes"]
 
@@ -230,6 +236,7 @@ Resources:
         IncludeTransitGatewayModule: !Ref IncludeTransitGatewayModule
         IncludeServiceQuotasModule: !Ref IncludeServiceQuotasModule
         IncludeResilienceHubModule: !Ref IncludeResilienceHubModule
+
 
   DataCollectorOrgAccountModulesReadStackSet:
     Type: AWS::CloudFormation::StackSet
@@ -272,6 +279,8 @@ Resources:
           ParameterValue: !Ref IncludeServiceQuotasModule
         - ParameterKey: IncludeResilienceHubModule
           ParameterValue: !Ref IncludeResilienceHubModule
+        - ParameterKey: IncludeRdsMultitenantModule
+          ParameterValue: !Ref IncludeRdsMultitenantModule
       StackInstancesGroup:
         - DeploymentTargets:
             OrganizationalUnitIds: !Split [",", !Ref OrganizationalUnitIds]

data-collection/deploy/deploy-in-linked-account.yaml

@@ -22,6 +22,7 @@ Metadata:
         - IncludeTransitGatewayModule
         - IncludeServiceQuotasModule
         - IncludeResilienceHubModule
+        - IncludeRdsMultitenantModule
     ParameterLabels:
       DataCollectionAccountID:
         default: 'Data Collection Account ID'
@@ -49,6 +50,8 @@ Metadata:
         default: 'Include Service Quotas Module'
       IncludeResilienceHubModule:
         default: 'Include Resilience Hub Module'
+      IncludeRdsMultitenantModule:
+        default: 'Include RDS Multitenant Module'
 
 Parameters:
   DataCollectionAccountID:
@@ -112,6 +115,11 @@ Parameters:
     Description: Collects Resilience Hub data from your accounts
     AllowedValues: ['yes', 'no']
     Default: 'no'
+  IncludeRdsMultitenantModule:
+    Type: String
+    Description: Collects RDS Performance Insights data for multi-tenant cost allocation
+    AllowedValues: ['yes', 'no']
+    Default: 'no'
 
 Conditions:
   IncludeTAModulePolicy:                 !Equals [!Ref IncludeTAModule, "yes"]
@@ -124,6 +132,7 @@ Conditions:
   IncludeTransitGatewayModulePolicy:     !Equals [!Ref IncludeTransitGatewayModule, "yes"]
   IncludeServiceQuotasModulePolicy:      !Equals [!Ref IncludeServiceQuotasModule, "yes"]
   IncludeResilienceHubModulePolicy:      !Equals [!Ref IncludeResilienceHubModule, "yes"]
+  IncludeRdsMultitenantModulePolicy:     !Equals [!Ref IncludeRdsMultitenantModule, "yes"]
 
 Outputs:
   LambdaRole:
@@ -155,6 +164,7 @@ Resources:
                   - !Sub "arn:${AWS::Partition}:iam::${DataCollectionAccountID}:role/${ResourcePrefix}support-cases-LambdaRole"
                   - !Sub "arn:${AWS::Partition}:iam::${DataCollectionAccountID}:role/${ResourcePrefix}service-quotas-LambdaRole"
                   - !Sub "arn:${AWS::Partition}:iam::${DataCollectionAccountID}:role/${ResourcePrefix}resilience-hub-LambdaRole"
+                  - !Sub "arn:${AWS::Partition}:iam::${DataCollectionAccountID}:role/${ResourcePrefix}RDSMultitenant-Lambda-Role"
       Path: /
     Metadata:
       cfn_nag:
@@ -460,6 +470,31 @@ Resources:
             Resource: "*" # Wildcard required as actions do not support   resource-level permissions
       Roles:
         - Ref: LambdaRole
+    Metadata:
+      cfn_nag:
+        rules_to_suppress:
+          - id: W12
+            reason: "Policy is used for scanning of a wide range of resources"
+  # RDS Multitenant policy
+  RdsMultitenantPolicy:
+    Type: 'AWS::IAM::Policy'
+    Condition: IncludeRdsMultitenantModulePolicy
+    Properties:
+      PolicyName: RdsMultitenantPolicy
+      PolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Effect: "Allow"
+            Action:
+              - "rds:DescribeDBInstances"
+            Resource: !Sub "arn:${AWS::Partition}:rds:*:${AWS::AccountId}:db:*"
+          - Effect: "Allow"
+            Action:
+              - "pi:GetResourceMetrics"
+              - "ec2:DescribeRegions"
+            Resource: "*"
+      Roles:
+        - Ref: LambdaRole
     Metadata:
       cfn_nag:
         rules_to_suppress:

data-collection/deploy/deploy-in-management-account.yaml

@@ -18,6 +18,7 @@ Metadata:
           - IncludeHealthEventsModule
           - IncludeRightsizingModule
           - IncludeLicenseManagerModule
+          - IncludeRdsMultitenantModule
           - IncludeServiceQuotasModule
     ParameterLabels:
       ManagementAccountRole:
@@ -38,6 +39,8 @@ Metadata:
         default: "Include Health Events Module"
       IncludeLicenseManagerModule:
         default: "Include Marketplace Licensing Module"
+      IncludeRdsMultitenantModule:
+        default: "Include RDS Multi-tenant Module"
       IncludeServiceQuotasModule:
         default: "Include Service Quotas Module"
 Parameters:
@@ -82,6 +85,11 @@ Parameters:
     Description: Collects Marketplace Licensing Information from your accounts
     AllowedValues: ['yes', 'no']
     Default: 'no'
+  IncludeRdsMultitenantModule:
+    Type: String
+    Description: Collects RDS Multi-tenant Performance Insights data from your accounts
+    AllowedValues: ['yes', 'no']
+    Default: 'no'
   IncludeServiceQuotasModule:
     Type: String
     Description: Collects Service Quotas Information from your accounts
@@ -95,6 +103,7 @@ Conditions:
   EnableBackupModule: !Equals [!Ref IncludeBackupModule, "yes"]
   EnableHealthEventsModule: !Equals [!Ref IncludeHealthEventsModule, "yes"]
   EnableLicenseManagerModule: !Equals [!Ref IncludeLicenseManagerModule, "yes"]
+  EnableRdsMultitenantModule: !Equals [!Ref IncludeRdsMultitenantModule, "yes"]
   EnableServiceQuotasModule: !Equals [!Ref IncludeServiceQuotasModule, "yes"]
 
 Outputs:
@@ -128,6 +137,7 @@ Resources:
                   - !Sub "arn:${AWS::Partition}:iam::${DataCollectionAccountID}:role/${ResourcePrefix}backup-LambdaRole"
                   - !Sub "arn:${AWS::Partition}:iam::${DataCollectionAccountID}:role/${ResourcePrefix}health-events-LambdaRole"
                   - !Sub "arn:${AWS::Partition}:iam::${DataCollectionAccountID}:role/${ResourcePrefix}license-manager-LambdaRole"
+                  - !Sub "arn:${AWS::Partition}:iam::${DataCollectionAccountID}:role/${ResourcePrefix}rds-multitenant-LambdaRole"
                   - !Sub "arn:${AWS::Partition}:iam::${DataCollectionAccountID}:role/${ResourcePrefix}RLS-LambdaRole"
                   - !Sub "arn:${AWS::Partition}:iam::${DataCollectionAccountID}:role/${ResourcePrefix}service-quotas-LambdaRole"
       Path: /
@@ -339,6 +349,29 @@ Resources:
         rules_to_suppress:
           - id: W12
             reason: "Policy is used for scanning of a wide range of resources"
+  RdsMultitenantPolicy:
+    Type: "AWS::IAM::Policy"
+    Condition: EnableRdsMultitenantModule
+    Properties:
+      PolicyName: RdsMultitenantPolicy
+      PolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Effect: "Allow"
+            Action:
+              - "pi:GetResourceMetrics"
+              - "pi:DescribeDimensionKeys"
+              - "pi:GetDimensionKeyDetails"
+              - "rds:DescribeDBInstances"
+              - "rds:DescribeDBClusters"
+            Resource: "*"
+      Roles:
+        - Ref: LambdaRole
+    Metadata:
+      cfn_nag:
+        rules_to_suppress:
+          - id: W12
+            reason: "Policy is used for scanning of a wide range of resources"
   ServiceQuotasPolicy:
     Type: "AWS::IAM::Policy"
     Condition: EnableServiceQuotasModule