Add lambda inventory (#142)

Co-authored-by: erikpeteaws <[email protected]>

Author: iakov-aws

data-collection/README.md

@@ -26,8 +26,8 @@ List of modules and objects collected:
 | `trusted-advisor`            | AWS Trusted Advisor   | Linked Account      | Requires Enterpriso or OnRamp Support Level |
 | `cost-explorer-cost-anomaly` | AWS Anomalies         | Management Account  |          |
 | `cost-explorer-rightsizing`  | AWS Cost Explorer     | Management Account  | DEPRECATED. Please use `cost-optimization-hub` |
-| `inventory`                  | Various services      | Linked Account      | Collects `Amazon OpenSearch Domains`, `Amazon ElastiCache Clusters`, `RDS DB Instances`, `EBS Volumes`, `AMI`, `EC2 Instances`, `EBS Snapshot`, `RDS Snapshot` |
-| `pricing`                    | Various services      | N/A                 | Collects pricing for `Amazon RDS`, `Amazon EC2`, `Amazon ElastiCache`, `Amazon OpenSearch`, `AWS Compute Savings Plan` |
+| `inventory`                  | Various services      | Linked Account      | Collects `Amazon OpenSearch Domains`, `Amazon ElastiCache Clusters`, `RDS DB Instances`, `EBS Volumes`, `AMI`, `EC2 Instances`, `EBS Snapshot`, `RDS Snapshot`, `Lambda` |
+| `pricing`                    | Various services      | N/A                 | Collects pricing for `Amazon RDS`, `Amazon EC2`, `Amazon ElastiCache`, `AWS Lambda`, `Amazon OpenSearch`, `AWS Compute Savings Plan` |
 | `rds-usage`                  |  Amazon RDS           | Linked Account      | Collects CloudWatch metrics for chargeback |
 | `transit-gateway`            |  AWS Transit Gateway  | Linked Account      | Collects CloudWatch metrics for chargeback |
 | `ecs-chargeback`             |  Amazon ECS           | Linked Account      |  |

data-collection/deploy/deploy-in-linked-account.yaml

@@ -197,6 +197,7 @@ Resources:
               - "elasticache:DescribeCacheClusters"
               - "eks:ListClusters"
               - "eks:DescribeCluster"
+              - "lambda:ListFunctions"
             Resource: "*" ## Policy is used for scanning of a wide range of resources
       Roles:
         - Ref: LambdaRole
@@ -272,4 +273,4 @@ Resources:
       cfn_nag:
         rules_to_suppress:
           - id: W12
-            reason: "Policy is used for scanning of a wide range of resources"
+            reason: "Policy is used for scanning of a wide range of resources"

data-collection/deploy/module-inventory.yaml

@@ -56,7 +56,7 @@ Parameters:
     Description: ARN of a Lambda for Managing GlueTable
   AwsObjects:
     Type: CommaDelimitedList
-    Default: OpensearchDomains, ElasticacheClusters, RdsDbInstances, EBS, AMI, Snapshot, Ec2Instances, VpcInstances, RdsDbSnapshots, EKSClusters
+    Default: OpensearchDomains, ElasticacheClusters, RdsDbInstances, EBS, AMI, Snapshot, Ec2Instances, VpcInstances, RdsDbSnapshots, EKSClusters, AWSLambda
     Description: Services for pulling price data
 
 Mappings:
@@ -789,7 +789,77 @@ Mappings:
                 paths: Arn,Name,CreatedAt,Version,accountid,collection_date,region
               SerializationLibrary: org.openx.data.jsonserde.JsonSerDe
           TableType: EXTERNAL_TABLE
-
+    AWSLambda:
+      path: lambda
+      table:
+        - Name: inventory_lambda_data
+          PartitionKeys:
+          - Name: payer_id
+            Type: string
+          - Name: year
+            Type: string
+          - Name: month
+            Type: string
+          - Name: day
+            Type: string
+          StorageDescriptor:
+            Columns:
+            - Name: functionname
+              Type: string
+            - Name: functionarn
+              Type: string
+            - Name: runtime
+              Type: string
+            - Name: role
+              Type: string
+            - Name: handler
+              Type: string
+            - Name: codesize
+              Type: int
+            - Name: description
+              Type: string
+            - Name: timeout
+              Type: int
+            - Name: memorysize
+              Type: int
+            - Name: lastmodified
+              Type: string
+            - Name: codesha256
+              Type: string
+            - Name: version
+              Type: string
+            - Name: tracingconfig
+              Type: struct<mode:string>
+            - Name: revisionid
+              Type: string
+            - Name: packagetype
+              Type: string
+            - Name: architectures
+              Type: array<string>
+            - Name: ephemeralstorage
+              Type: struct<size:int>
+            - Name: snapstart
+              Type: struct<applyon:string,optimizationstatus:string>
+            - Name: loggingconfig
+              Type: struct<logformat:string,loggroup:string>
+            - Name: accountid
+              Type: string
+            - Name: collection_date
+              Type: string
+            - Name: region
+              Type: string
+            - Name: layers
+              Type: array<struct<arn:string,codesize:int>>
+            - Name: vpcconfig
+              Type: struct<subnetids:array<string>,securitygroupids:array<string>,vpcid:string,ipv6allowedfordualstack:boolean>
+            InputFormat: org.apache.hadoop.mapred.TextInputFormat
+            Location: !Sub s3://${DestinationBucket}/inventory/inventory-lambda-data/
+            OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+            SerdeInfo:
+              Parameters:
+                paths: functionname, functionarn, runtime, role, handler, codesize, dedscription, timeout, memorysize, lastmodified, codesha256, version, tracingconfig, revisionid, packagetype, architectures, ephemeralstorage, snapstart, loggingconfig, accountid, collection_date, region, layers, vpcconfig
+              SerializationLibrary: org.openx.data.jsonserde.JsonSerDe
+          TableType: EXTERNAL_TABLE
 
 Resources:
   LambdaRole:
@@ -1005,6 +1075,12 @@ Resources:
                       service='ec2',
                       function_name='describe_vpcs'
                   ),
+                  'lambda' : partial(
+                    paginated_scan,
+                    service='lambda',
+                    function_name='list_functions',
+                    obj_name='Functions[*]'
+                  ),
                   'eks': eks_clusters_scan
               }
 

data-collection/deploy/module-pricing.yaml

@@ -35,7 +35,7 @@ Parameters:
     Description: Arn of lambda for Analytics
   AwsServices:
     Type: CommaDelimitedList
-    Default: AmazonRDS, AmazonEC2, AmazonElastiCache, AmazonES, AWSComputeSavingsPlan, RegionNames
+    Default: AmazonRDS, AmazonEC2, AmazonElastiCache, AmazonES, AWSComputeSavingsPlan, RegionNames, AWSLambda
     Description: Services for pulling price data
   CodeBucket:
     Type: String
@@ -317,6 +317,34 @@ Mappings:
           - { Type: string, Name: regionname }
           - { Type: string, Name: endpoint }
           - { Type: string, Name: protocol }
+    AWSLambda:
+      path: lambda
+      partition: 
+          - { Name: region, Type: string }
+      fields:
+          - { Type: string, Name: sku }
+          - { Type: string, Name: offertermcode }
+          - { Type: string, Name: ratecode }
+          - { Type: string, Name: termtype }
+          - { Type: string, Name: pricedescription }
+          - { Type: string, Name: effectivedate }
+          - { Type: string, Name: endingrange }
+          - { Type: bigint, Name: startingrange }
+          - { Type: string, Name: endingrange }
+          - { Type: string, Name: unit }
+          - { Type: double, Name: priceperunit }
+          - { Type: string, Name: currency }
+          - { Type: string, Name: relatedto }
+          - { Type: string, Name: product family }
+          - { Type: string, Name: servicecode }
+          - { Type: string, Name: location }
+          - { Type: string, Name: location type }
+          - { Type: string, Name: group }
+          - { Type: string, Name: group description }
+          - { Type: string, Name: usagetype }
+          - { Type: string, Name: operation }
+          - { Type: string, Name: region code }
+          - { Type: string, Name: servicename }
     RDSGraviton:
       path: rdsgraviton
       partition:
@@ -695,4 +723,4 @@ Resources:
     Type: Custom::LambdaAnalyticsExecutor
     Properties:
       ServiceToken: !Ref LambdaAnalyticsARN
-      Name: !Ref CFDataName
+      Name: !Ref CFDataName

data-collection/test/test_from_scratch.py

@@ -83,7 +83,11 @@ def test_inventory_vpc_data(athena):
 
 def test_inventory_rds_snaphot_data(athena):
     data = athena_query(athena=athena, sql_query='SELECT * FROM "optimization_data"."inventory_rds_db_snapshots_data" LIMIT 10;')
-    assert len(data) > 0, 'inventory_vpc_data is empty'
+    assert len(data) > 0, 'inventory_rds_db_snapshots_data is empty'
+
+def test_inventory_lambda_data(athena):
+    data = athena_query(athena=athena, sql_query='SELECT * FROM "optimization_data"."inventory_lambda_data" LIMIT 10;')
+    assert len(data) > 0, 'inventory_lambda_data is empty'
 
 def test_rds_usage_data(athena):
     data = athena_query(athena=athena, sql_query='SELECT * FROM "optimization_data"."rds_usage_data" LIMIT 10;')
@@ -137,6 +141,10 @@ def test_pricing_rds_data(athena):
     data = athena_query(athena=athena, sql_query='SELECT * FROM "optimization_data"."pricing_rds_data" LIMIT 10;')
     assert len(data) > 0, 'pricing_rds_data is empty'
 
+def test_pricing_lambda_data(athena):
+    data = athena_query(athena=athena, sql_query='SELECT * FROM "optimization_data"."pricing_lambda_data" LIMIT 10;')
+    assert len(data) > 0, 'pricing_awslambda_data is empty'
+
 def test_pricing_regionnames_data(athena):
     data = athena_query(athena=athena, sql_query='SELECT * FROM "optimization_data"."pricing_regionnames_data" LIMIT 10;')
     assert len(data) > 0, 'pricing_regionnames_data is empty'
@@ -164,4 +172,4 @@ def test_health_events_data(athena):
 
 
 if __name__ == '__main__':
-    pytest.main()
+    pytest.main()

data-collection/test/utils.py

@@ -344,6 +344,7 @@ def trigger_update(account_id):
         f'arn:aws:states:{region}:{account_id}:stateMachine:{PREFIX}inventory-Ec2Instances-StateMachine',
         f'arn:aws:states:{region}:{account_id}:stateMachine:{PREFIX}inventory-VpcInstances-StateMachine',
         f'arn:aws:states:{region}:{account_id}:stateMachine:{PREFIX}inventory-RdsDbSnapshots-StateMachine',
+        f'arn:aws:states:{region}:{account_id}:stateMachine:{PREFIX}inventory-AWSLambda-StateMachine',
         f'arn:aws:states:{region}:{account_id}:stateMachine:{PREFIX}rds-usage-StateMachine',
         f'arn:aws:states:{region}:{account_id}:stateMachine:{PREFIX}transit-gateway-StateMachine',
         f'arn:aws:states:{region}:{account_id}:stateMachine:{PREFIX}trusted-advisor-StateMachine',
@@ -356,6 +357,7 @@ def trigger_update(account_id):
         f"arn:aws:states:{region}:{account_id}:stateMachine:{PREFIX}pricing-AmazonElastiCache-StateMachine",
         f"arn:aws:states:{region}:{account_id}:stateMachine:{PREFIX}pricing-AmazonES-StateMachine",
         f"arn:aws:states:{region}:{account_id}:stateMachine:{PREFIX}pricing-AWSComputeSavingsPlan-StateMachine",
+        f"arn:aws:states:{region}:{account_id}:stateMachine:{PREFIX}pricing-AWSLambda-StateMachine",
         f"arn:aws:states:{region}:{account_id}:stateMachine:{PREFIX}pricing-RegionNames-StateMachine",
         f"arn:aws:states:{region}:{account_id}:stateMachine:{PREFIX}backup-CopyJobs-StateMachine",
         f"arn:aws:states:{region}:{account_id}:stateMachine:{PREFIX}backup-RestoreJobs-StateMachine",
@@ -414,4 +416,4 @@ def cleanup_stacks(cloudformation, account_id, s3, s3client, athena, glue):
 def prepare_stacks(cloudformation, account_id, org_unit_id, s3, s3client, bucket):
     initial_deploy_stacks(cloudformation=cloudformation, account_id=account_id, org_unit_id=org_unit_id, bucket=bucket)
     clean_bucket(s3=s3, s3client=s3client,  account_id=account_id, full=False)
-    trigger_update(account_id=account_id)
+    trigger_update(account_id=account_id)