Skip to content

Commit ae242c1

Browse files
iakov-awsiakov-aws
committed
fix hive issues
1 parent b5700bf commit ae242c1

18 files changed

+379
-37
lines changed

‎data-collection/deploy/module-aws-feeds.yaml‎

Lines changed: 76 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -716,8 +716,25 @@ Resources:
716716
S3Targets:
717717
- Path: !Sub "s3://${DestinationBucket}/aws-feeds/aws-feeds-whats-new/"
718718
- Path: !Sub "s3://${DestinationBucket}/aws-feeds/aws-cid-feeds-whats-new/"
719-
Configuration: "{\"Version\":1.0,\"CrawlerOutput\":{\"Partitions\":{\"AddOrUpdateBehavior\":\"InheritFromTable\"}}}"
720-
719+
Configuration: |
720+
{
721+
"Version": 1.0,
722+
"Grouping": {
723+
"TableGroupingPolicy": "CombineCompatibleSchemas"
724+
},
725+
"CrawlerOutput": {
726+
"Partitions": {
727+
"AddOrUpdateBehavior": "InheritFromTable"
728+
},
729+
"Tables": {
730+
"TableThreshold": 2,
731+
"AddOrUpdateBehavior": "MergeNewColumns"
732+
}
733+
}
734+
}
735+
SchemaChangePolicy:
736+
UpdateBehavior: UPDATE_IN_DATABASE
737+
DeleteBehavior: LOG
721738
CrawlerBlogPost:
722739
Type: AWS::Glue::Crawler
723740
Properties:
@@ -727,8 +744,25 @@ Resources:
727744
Targets:
728745
S3Targets:
729746
- Path: !Sub "s3://${DestinationBucket}/aws-feeds/aws-feeds-blog-post/"
730-
Configuration: "{\"Version\":1.0,\"CrawlerOutput\":{\"Partitions\":{\"AddOrUpdateBehavior\":\"InheritFromTable\"}}}"
731-
747+
Configuration: |
748+
{
749+
"Version": 1.0,
750+
"Grouping": {
751+
"TableGroupingPolicy": "CombineCompatibleSchemas"
752+
},
753+
"CrawlerOutput": {
754+
"Partitions": {
755+
"AddOrUpdateBehavior": "InheritFromTable"
756+
},
757+
"Tables": {
758+
"TableThreshold": 1,
759+
"AddOrUpdateBehavior": "MergeNewColumns"
760+
}
761+
}
762+
}
763+
SchemaChangePolicy:
764+
UpdateBehavior: UPDATE_IN_DATABASE
765+
DeleteBehavior: LOG
732766
CrawlerYoutTube:
733767
Type: AWS::Glue::Crawler
734768
Properties:
@@ -738,8 +772,25 @@ Resources:
738772
Targets:
739773
S3Targets:
740774
- Path: !Sub "s3://${DestinationBucket}/aws-feeds/aws-feeds-youtube/"
741-
Configuration: "{\"Version\":1.0,\"CrawlerOutput\":{\"Partitions\":{\"AddOrUpdateBehavior\":\"InheritFromTable\"}}}"
742-
775+
Configuration: |
776+
{
777+
"Version": 1.0,
778+
"Grouping": {
779+
"TableGroupingPolicy": "CombineCompatibleSchemas"
780+
},
781+
"CrawlerOutput": {
782+
"Partitions": {
783+
"AddOrUpdateBehavior": "InheritFromTable"
784+
},
785+
"Tables": {
786+
"TableThreshold": 1,
787+
"AddOrUpdateBehavior": "MergeNewColumns"
788+
}
789+
}
790+
}
791+
SchemaChangePolicy:
792+
UpdateBehavior: UPDATE_IN_DATABASE
793+
DeleteBehavior: LOG
743794
CrawlerSecurityBulletin:
744795
Type: AWS::Glue::Crawler
745796
Properties:
@@ -749,8 +800,25 @@ Resources:
749800
Targets:
750801
S3Targets:
751802
- Path: !Sub "s3://${DestinationBucket}/aws-feeds/aws-feeds-security-bulletin/"
752-
Configuration: "{\"Version\":1.0,\"CrawlerOutput\":{\"Partitions\":{\"AddOrUpdateBehavior\":\"InheritFromTable\"}}}"
753-
803+
Configuration: |
804+
{
805+
"Version": 1.0,
806+
"Grouping": {
807+
"TableGroupingPolicy": "CombineCompatibleSchemas"
808+
},
809+
"CrawlerOutput": {
810+
"Partitions": {
811+
"AddOrUpdateBehavior": "InheritFromTable"
812+
},
813+
"Tables": {
814+
"TableThreshold": 1,
815+
"AddOrUpdateBehavior": "MergeNewColumns"
816+
}
817+
}
818+
}
819+
SchemaChangePolicy:
820+
UpdateBehavior: UPDATE_IN_DATABASE
821+
DeleteBehavior: LOG
754822
ModuleStepFunctionWhatsNew:
755823
Type: AWS::StepFunctions::StateMachine
756824
Properties:

‎data-collection/deploy/module-budgets.yaml‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -270,7 +270,25 @@ Resources:
270270
Targets:
271271
S3Targets:
272272
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-data/"
273-
273+
Configuration: |
274+
{
275+
"Version": 1.0,
276+
"Grouping": {
277+
"TableGroupingPolicy": "CombineCompatibleSchemas"
278+
},
279+
"CrawlerOutput": {
280+
"Partitions": {
281+
"AddOrUpdateBehavior": "InheritFromTable"
282+
},
283+
"Tables": {
284+
"TableThreshold": 1,
285+
"AddOrUpdateBehavior": "MergeNewColumns"
286+
}
287+
}
288+
}
289+
SchemaChangePolicy:
290+
UpdateBehavior: UPDATE_IN_DATABASE
291+
DeleteBehavior: LOG
274292
ModuleStepFunction:
275293
Type: AWS::StepFunctions::StateMachine
276294
Properties:

‎data-collection/deploy/module-cost-anomaly.yaml‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -341,12 +341,18 @@ Resources:
341341
"TableGroupingPolicy": "CombineCompatibleSchemas"
342342
},
343343
"CrawlerOutput": {
344+
"Partitions": {
345+
"AddOrUpdateBehavior": "InheritFromTable"
346+
},
344347
"Tables": {
345-
"TableThreshold": 1
348+
"TableThreshold": 1,
349+
"AddOrUpdateBehavior": "MergeNewColumns"
346350
}
347351
}
348352
}
349-
353+
SchemaChangePolicy:
354+
UpdateBehavior: UPDATE_IN_DATABASE
355+
DeleteBehavior: LOG
350356
ModuleGlueTable:
351357
Type: Custom::ManageGlueTable
352358
Properties:

‎data-collection/deploy/module-cost-explorer-rightsizing.yaml‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -242,7 +242,25 @@ Resources:
242242
Targets:
243243
S3Targets:
244244
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-data/"
245-
245+
Configuration: |
246+
{
247+
"Version": 1.0,
248+
"Grouping": {
249+
"TableGroupingPolicy": "CombineCompatibleSchemas"
250+
},
251+
"CrawlerOutput": {
252+
"Partitions": {
253+
"AddOrUpdateBehavior": "InheritFromTable"
254+
},
255+
"Tables": {
256+
"TableThreshold": 1,
257+
"AddOrUpdateBehavior": "MergeNewColumns"
258+
}
259+
}
260+
}
261+
SchemaChangePolicy:
262+
UpdateBehavior: UPDATE_IN_DATABASE
263+
DeleteBehavior: LOG
246264
ModuleStepFunction:
247265
Type: AWS::StepFunctions::StateMachine
248266
Properties:

‎data-collection/deploy/module-ecs-chargeback.yaml‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -254,7 +254,25 @@ Resources:
254254
Targets:
255255
S3Targets:
256256
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-data/"
257-
257+
Configuration: |
258+
{
259+
"Version": 1.0,
260+
"Grouping": {
261+
"TableGroupingPolicy": "CombineCompatibleSchemas"
262+
},
263+
"CrawlerOutput": {
264+
"Partitions": {
265+
"AddOrUpdateBehavior": "InheritFromTable"
266+
},
267+
"Tables": {
268+
"TableThreshold": 1,
269+
"AddOrUpdateBehavior": "MergeNewColumns"
270+
}
271+
}
272+
}
273+
SchemaChangePolicy:
274+
UpdateBehavior: UPDATE_IN_DATABASE
275+
DeleteBehavior: LOG
258276
ModuleStepFunction:
259277
Type: AWS::StepFunctions::StateMachine
260278
Properties:

‎data-collection/deploy/module-health-events.yaml‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -466,7 +466,25 @@ Resources:
466466
Targets:
467467
S3Targets:
468468
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-detail-data/"
469-
469+
Configuration: |
470+
{
471+
"Version": 1.0,
472+
"Grouping": {
473+
"TableGroupingPolicy": "CombineCompatibleSchemas"
474+
},
475+
"CrawlerOutput": {
476+
"Partitions": {
477+
"AddOrUpdateBehavior": "InheritFromTable"
478+
},
479+
"Tables": {
480+
"TableThreshold": 1,
481+
"AddOrUpdateBehavior": "MergeNewColumns"
482+
}
483+
}
484+
}
485+
SchemaChangePolicy:
486+
UpdateBehavior: UPDATE_IN_DATABASE
487+
DeleteBehavior: LOG
470488
ModuleStepFunction:
471489
Type: AWS::StepFunctions::StateMachine
472490
Properties:

‎data-collection/deploy/module-isv-feeds.yaml‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -763,8 +763,25 @@ Resources:
763763
- Path: !Sub s3://${DestinationBucket}/isv-feeds/databricks-feeds-whats-new/
764764
- Path: !Sub s3://${DestinationBucket}/isv-feeds/gitlab-feeds-whats-new/
765765
- Path: !Sub s3://${DestinationBucket}/isv-feeds/circle-ci-feeds-whats-new/
766-
Configuration: '{"Version":1.0,"CrawlerOutput":{"Partitions":{"AddOrUpdateBehavior":"InheritFromTable"}}}'
767-
766+
Configuration: |
767+
{
768+
"Version": 1.0,
769+
"Grouping": {
770+
"TableGroupingPolicy": "CombineCompatibleSchemas"
771+
},
772+
"CrawlerOutput": {
773+
"Partitions": {
774+
"AddOrUpdateBehavior": "InheritFromTable"
775+
},
776+
"Tables": {
777+
"TableThreshold": 9,
778+
"AddOrUpdateBehavior": "MergeNewColumns"
779+
}
780+
}
781+
}
782+
SchemaChangePolicy:
783+
UpdateBehavior: UPDATE_IN_DATABASE
784+
DeleteBehavior: LOG
768785
StepFunction:
769786
Type: AWS::StepFunctions::StateMachine
770787
Properties:

‎data-collection/deploy/module-license-manager.yaml‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -276,7 +276,25 @@ Resources:
276276
Targets:
277277
S3Targets:
278278
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-${GrantDataPrefix}/"
279-
279+
Configuration: |
280+
{
281+
"Version": 1.0,
282+
"Grouping": {
283+
"TableGroupingPolicy": "CombineCompatibleSchemas"
284+
},
285+
"CrawlerOutput": {
286+
"Partitions": {
287+
"AddOrUpdateBehavior": "InheritFromTable"
288+
},
289+
"Tables": {
290+
"TableThreshold": 1,
291+
"AddOrUpdateBehavior": "MergeNewColumns"
292+
}
293+
}
294+
}
295+
SchemaChangePolicy:
296+
UpdateBehavior: UPDATE_IN_DATABASE
297+
DeleteBehavior: LOG
280298
LicensesCrawler:
281299
Type: AWS::Glue::Crawler
282300
Properties:
@@ -286,7 +304,25 @@ Resources:
286304
Targets:
287305
S3Targets:
288306
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-${LicenseDataPrefix}/"
289-
307+
Configuration: |
308+
{
309+
"Version": 1.0,
310+
"Grouping": {
311+
"TableGroupingPolicy": "CombineCompatibleSchemas"
312+
},
313+
"CrawlerOutput": {
314+
"Partitions": {
315+
"AddOrUpdateBehavior": "InheritFromTable"
316+
},
317+
"Tables": {
318+
"TableThreshold": 1,
319+
"AddOrUpdateBehavior": "MergeNewColumns"
320+
}
321+
}
322+
}
323+
SchemaChangePolicy:
324+
UpdateBehavior: UPDATE_IN_DATABASE
325+
DeleteBehavior: LOG
290326
ModuleStepFunction:
291327
Type: AWS::StepFunctions::StateMachine
292328
Properties:

‎data-collection/deploy/module-organization.yaml‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -304,7 +304,25 @@ Resources:
304304
Targets:
305305
S3Targets:
306306
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/organization-data/" #Preserve legacy name in Athena
307-
307+
Configuration: |
308+
{
309+
"Version": 1.0,
310+
"Grouping": {
311+
"TableGroupingPolicy": "CombineCompatibleSchemas"
312+
},
313+
"CrawlerOutput": {
314+
"Partitions": {
315+
"AddOrUpdateBehavior": "InheritFromTable"
316+
},
317+
"Tables": {
318+
"TableThreshold": 1,
319+
"AddOrUpdateBehavior": "MergeNewColumns"
320+
}
321+
}
322+
}
323+
SchemaChangePolicy:
324+
UpdateBehavior: UPDATE_IN_DATABASE
325+
DeleteBehavior: LOG
308326
ModuleStepFunction:
309327
Type: AWS::StepFunctions::StateMachine
310328
Properties:

‎data-collection/deploy/module-quicksight.yaml‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,25 @@ Resources:
312312
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-groupmembership-data/"
313313
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-group-data/"
314314
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-user-data/"
315-
315+
Configuration: |
316+
{
317+
"Version": 1.0,
318+
"Grouping": {
319+
"TableGroupingPolicy": "CombineCompatibleSchemas"
320+
},
321+
"CrawlerOutput": {
322+
"Partitions": {
323+
"AddOrUpdateBehavior": "InheritFromTable"
324+
},
325+
"Tables": {
326+
"TableThreshold": 3,
327+
"AddOrUpdateBehavior": "MergeNewColumns"
328+
}
329+
}
330+
}
331+
SchemaChangePolicy:
332+
UpdateBehavior: UPDATE_IN_DATABASE
333+
DeleteBehavior: LOG
316334
ModuleStepFunction:
317335
Type: AWS::StepFunctions::StateMachine
318336
Properties:

0 commit comments

Comments
 (0)