Crawler settings (#157)

* typos

* fix crawler

* fix crawler

* fix crawler

* minor fixes and adding test option

* lint

Author: iakov-aws

data-collection/CONTRIBUTING.md

@@ -69,13 +69,10 @@ python3 ./data-collection/utils/pylint.py
 3. Upload the code to a bucket and run integration tests in your testing environment
 
 ```bash
-export account_id=$(aws sts get-caller-identity --query "Account" --output text )
-export bucket=cid-$account_id-test
-./data-collection/utils/upload.sh  "$bucket"
-python3 ./data-collection/test/test_from_scratch.py
+./data-collection/test/run-test-from-scratch.sh --no-teardown
 ```
 
-The test will install stacks from scratch in a single account, then it will check the presence of Athena tables. After running tests, it will delete the stacks and all artefacts that are not deleted by CFN.
+The test will install stacks from scratch in a single account, then it will check the presence of Athena tables. After running tests, it will delete the stacks and all artifacts that are not deleted by CFN. You can avoid teardown by providing a flag `--no-teardown`.
 
 4. Create a merge request.
 

data-collection/deploy/module-inventory.yaml

@@ -65,6 +65,7 @@ Mappings:
       path: opensearch-domains
       table:
         - Name: inventory_opensearch_domains_data
+          Parameters: { "classification" : "json", "compressionType": "none" }
           PartitionKeys:
           - Name: payer_id
             Type: string
@@ -105,6 +106,7 @@ Mappings:
       path: elasticache-clusters
       table:
         - Name: inventory_elasticache_clusters_data
+          Parameters: { "classification" : "json", "compressionType": "none" }
           PartitionKeys:
           - Name: payer_id
             Type: string
@@ -192,7 +194,7 @@ Mappings:
       table:
         - Name: inventory_rds_db_instances_data
           TableType: EXTERNAL_TABLE
-
+          Parameters: { "classification" : "json", "compressionType": "none" }
           PartitionKeys:
           - { Name: payer_id, Type: string }
           - { Name: year, Type: string }
@@ -334,6 +336,7 @@ Mappings:
       path: rds-db-snapshots
       table:
         - Name: inventory_rds_db_snapshots_data
+          Parameters: { "classification" : "json", "compressionType": "none" }
           PartitionKeys:
           - Name: payer_id
             Type: string
@@ -422,6 +425,7 @@ Mappings:
       path: ebs
       table:
         - Name: inventory_ebs_data
+          Parameters: { "classification" : "json", "compressionType": "none" }
           PartitionKeys:
           - Name: payer_id
             Type: string
@@ -478,6 +482,7 @@ Mappings:
       path: ami
       table:
         - Name: inventory_ami_data
+          Parameters: { "classification" : "json", "compressionType": "none" }
           PartitionKeys:
           - Name: payer_id
             Type: string
@@ -550,6 +555,7 @@ Mappings:
       path: snapshot
       table:
         - Name: inventory_snapshot_data
+          Parameters: { "classification" : "json", "compressionType": "none" }
           PartitionKeys:
           - Name: payer_id
             Type: string
@@ -602,6 +608,7 @@ Mappings:
       path: ec2-instances
       table:
         - Name: inventory_ec2_instances_data
+          Parameters: { "classification" : "json", "compressionType": "none" }
           PartitionKeys:
           - Name: payer_id
             Type: string
@@ -711,6 +718,7 @@ Mappings:
       path: vpc
       table:
         - Name: inventory_vpc_data
+          Parameters: { "classification" : "json", "compressionType": "none" }
           PartitionKeys:
           - Name: payer_id
             Type: string
@@ -756,6 +764,7 @@ Mappings:
       path: eks
       table:
         - Name: inventory_eks_data
+          Parameters: { "classification" : "json", "compressionType": "none" }
           PartitionKeys:
           - Name: payer_id
             Type: string
@@ -793,6 +802,7 @@ Mappings:
       path: lambda
       table:
         - Name: inventory_lambda_data
+          Parameters: { "classification" : "json", "compressionType": "none" }
           PartitionKeys:
           - Name: payer_id
             Type: string
@@ -849,7 +859,7 @@ Mappings:
             - Name: region
               Type: string
             - Name: layers
-              Type: array<struct<arn:string,codesize:int>>
+              Type: array<struct<arn:string,codesize:int>> # will be updated
             - Name: vpcconfig
               Type: struct<subnetids:array<string>,securitygroupids:array<string>,vpcid:string,ipv6allowedfordualstack:boolean>
             InputFormat: org.apache.hadoop.mapred.TextInputFormat
@@ -1161,9 +1171,12 @@ Resources:
           Configuration: |
             {
               "Version": 1.0,
+              "Grouping": {
+                "TableGroupingPolicy": "CombineCompatibleSchemas"
+              },
               "CrawlerOutput": {
-                "Partitions": {
-                  "AddOrUpdateBehavior": "InheritFromTable"
+                "Tables": {
+                  "TableThreshold": 1
                 }
               }
             }

data-collection/test/conftest.py

@@ -89,5 +89,7 @@ def mode(request):
 @pytest.fixture(scope='session', autouse=True)
 def prepare_setup(athena, cloudformation, s3, s3client, account_id, org_unit_id, bucket, start_time, mode, glue):
     yield prepare_stacks(cloudformation=cloudformation, account_id=account_id, org_unit_id=org_unit_id, bucket=bucket, s3=s3, s3client=s3client)
+
+    mode = pytest.params.get('mode', mode)
     if mode != "no-teardown":
         cleanup_stacks(cloudformation=cloudformation, account_id=account_id, s3=s3, s3client=s3client, athena=athena, glue=glue)

data-collection/test/run-test-from-scratch.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+# see ../CONTRIBUTION.md
+
+# vars
+account_id=$(aws sts get-caller-identity --query "Account" --output text )
+bucket=cid-$account_id-test
+
+# upload files
+./data-collection/utils/upload.sh  "$bucket"
+
+# run test
+python3 ./data-collection/test/test_from_scratch.py "$@"

data-collection/test/test_from_scratch.py

@@ -31,6 +31,7 @@
 
 """
 import logging
+import sys
 
 import pytest
 
@@ -180,4 +181,10 @@ def test_license_manager_licenses(athena):
 
 
 if __name__ == '__main__':
+    pytest.params = {}
+    if '--no-teardown' in sys.argv:
+        sys.argv.remove('--no-teardown')
+        pytest.params['mode'] = 'no-teardown'
+
+    sys.argv = sys.argv[:1]
     pytest.main()