Skip to content

Commit 9253f81

Browse files
Changes include (1) Excel as S3 Source type and XML and Tableau's Hyper as S3 Sink types, (2) targeted number of partitions parameter in S3 sinks and (3) new compression types in CSV/JSON and Parquet S3 sinks.
1 parent a9a14fe commit 9253f81

29 files changed

+2252
-4
lines changed

generator/ServiceModels/glue/glue-2017-03-31.api.json

Lines changed: 86 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5375,6 +5375,7 @@
53755375
"RedshiftSource":{"shape":"RedshiftSource"},
53765376
"S3CatalogSource":{"shape":"S3CatalogSource"},
53775377
"S3CsvSource":{"shape":"S3CsvSource"},
5378+
"S3ExcelSource":{"shape":"S3ExcelSource"},
53785379
"S3JsonSource":{"shape":"S3JsonSource"},
53795380
"S3ParquetSource":{"shape":"S3ParquetSource"},
53805381
"RelationalCatalogSource":{"shape":"RelationalCatalogSource"},
@@ -5385,7 +5386,9 @@
53855386
"RedshiftTarget":{"shape":"RedshiftTarget"},
53865387
"S3CatalogTarget":{"shape":"S3CatalogTarget"},
53875388
"S3GlueParquetTarget":{"shape":"S3GlueParquetTarget"},
5389+
"S3HyperDirectTarget":{"shape":"S3HyperDirectTarget"},
53885390
"S3DirectTarget":{"shape":"S3DirectTarget"},
5391+
"S3IcebergDirectTarget":{"shape":"S3IcebergDirectTarget"},
53895392
"ApplyMapping":{"shape":"ApplyMapping"},
53905393
"SelectFields":{"shape":"SelectFields"},
53915394
"DropFields":{"shape":"DropFields"},
@@ -10575,6 +10578,10 @@
1057510578
"type":"list",
1057610579
"member":{"shape":"HudiTarget"}
1057710580
},
10581+
"HyperTargetCompressionType":{
10582+
"type":"string",
10583+
"enum":["uncompressed"]
10584+
},
1057810585
"IAMRoleArn":{
1057910586
"type":"string",
1058010587
"pattern":"^arn:aws(-(cn|us-gov|iso(-[bef])?))?:iam::[0-9]{12}:role/.+"
@@ -10641,6 +10648,15 @@
1064110648
"MaximumTraversalDepth":{"shape":"NullableInteger"}
1064210649
}
1064310650
},
10651+
"IcebergTargetCompressionType":{
10652+
"type":"string",
10653+
"enum":[
10654+
"gzip",
10655+
"lzo",
10656+
"uncompressed",
10657+
"snappy"
10658+
]
10659+
},
1064410660
"IcebergTargetList":{
1064510661
"type":"list",
1064610662
"member":{"shape":"IcebergTarget"}
@@ -12344,6 +12360,7 @@
1234412360
"type":"string",
1234512361
"box":true
1234612362
},
12363+
"NumberTargetPartitionsString":{"type":"string"},
1234712364
"OAuth2ClientApplication":{
1234812365
"type":"structure",
1234912366
"members":{
@@ -12644,6 +12661,8 @@
1264412661
"snappy",
1264512662
"lzo",
1264612663
"gzip",
12664+
"brotli",
12665+
"lz4",
1264712666
"uncompressed",
1264812667
"none"
1264912668
]
@@ -13647,6 +13666,7 @@
1364713666
"PartitionKeys":{"shape":"GlueStudioPathList"},
1364813667
"Path":{"shape":"EnclosedInStringProperty"},
1364913668
"Compression":{"shape":"DeltaTargetCompressionType"},
13669+
"NumberTargetPartitions":{"shape":"NumberTargetPartitionsString"},
1365013670
"Format":{"shape":"TargetFormat"},
1365113671
"AdditionalOptions":{"shape":"AdditionalOptions"},
1365213672
"SchemaChangePolicy":{"shape":"DirectSchemaChangePolicy"}
@@ -13689,6 +13709,7 @@
1368913709
"PartitionKeys":{"shape":"GlueStudioPathList"},
1369013710
"Path":{"shape":"EnclosedInStringProperty"},
1369113711
"Compression":{"shape":"EnclosedInStringProperty"},
13712+
"NumberTargetPartitions":{"shape":"NumberTargetPartitionsString"},
1369213713
"Format":{"shape":"TargetFormat"},
1369313714
"SchemaChangePolicy":{"shape":"DirectSchemaChangePolicy"}
1369413715
}
@@ -13712,6 +13733,28 @@
1371213733
"SSE-S3"
1371313734
]
1371413735
},
13736+
"S3ExcelSource":{
13737+
"type":"structure",
13738+
"required":[
13739+
"Name",
13740+
"Paths"
13741+
],
13742+
"members":{
13743+
"Name":{"shape":"NodeName"},
13744+
"Paths":{"shape":"EnclosedInStringProperties"},
13745+
"CompressionType":{"shape":"ParquetCompressionType"},
13746+
"Exclusions":{"shape":"EnclosedInStringProperties"},
13747+
"GroupSize":{"shape":"EnclosedInStringProperty"},
13748+
"GroupFiles":{"shape":"EnclosedInStringProperty"},
13749+
"Recurse":{"shape":"BoxedBoolean"},
13750+
"MaxBand":{"shape":"BoxedNonNegativeInt"},
13751+
"MaxFilesInBand":{"shape":"BoxedNonNegativeInt"},
13752+
"AdditionalOptions":{"shape":"S3DirectSourceAdditionalOptions"},
13753+
"NumberRows":{"shape":"BoxedLong"},
13754+
"SkipFooter":{"shape":"BoxedNonNegativeInt"},
13755+
"OutputSchemas":{"shape":"GlueSchemas"}
13756+
}
13757+
},
1371513758
"S3GlueParquetTarget":{
1371613759
"type":"structure",
1371713760
"required":[
@@ -13725,6 +13768,7 @@
1372513768
"PartitionKeys":{"shape":"GlueStudioPathList"},
1372613769
"Path":{"shape":"EnclosedInStringProperty"},
1372713770
"Compression":{"shape":"ParquetCompressionType"},
13771+
"NumberTargetPartitions":{"shape":"NumberTargetPartitionsString"},
1372813772
"SchemaChangePolicy":{"shape":"DirectSchemaChangePolicy"}
1372913773
}
1373013774
},
@@ -13762,6 +13806,7 @@
1376213806
"Inputs":{"shape":"OneInput"},
1376313807
"Path":{"shape":"EnclosedInStringProperty"},
1376413808
"Compression":{"shape":"HudiTargetCompressionType"},
13809+
"NumberTargetPartitions":{"shape":"NumberTargetPartitionsString"},
1376513810
"PartitionKeys":{"shape":"GlueStudioPathList"},
1376613811
"Format":{"shape":"TargetFormat"},
1376713812
"AdditionalOptions":{"shape":"AdditionalOptions"},
@@ -13782,6 +13827,43 @@
1378213827
"OutputSchemas":{"shape":"GlueSchemas"}
1378313828
}
1378413829
},
13830+
"S3HyperDirectTarget":{
13831+
"type":"structure",
13832+
"required":[
13833+
"Name",
13834+
"Inputs",
13835+
"Path"
13836+
],
13837+
"members":{
13838+
"Name":{"shape":"NodeName"},
13839+
"Inputs":{"shape":"OneInput"},
13840+
"PartitionKeys":{"shape":"GlueStudioPathList"},
13841+
"Path":{"shape":"EnclosedInStringProperty"},
13842+
"Compression":{"shape":"HyperTargetCompressionType"},
13843+
"SchemaChangePolicy":{"shape":"DirectSchemaChangePolicy"}
13844+
}
13845+
},
13846+
"S3IcebergDirectTarget":{
13847+
"type":"structure",
13848+
"required":[
13849+
"Name",
13850+
"Inputs",
13851+
"Path",
13852+
"Format",
13853+
"Compression"
13854+
],
13855+
"members":{
13856+
"Name":{"shape":"NodeName"},
13857+
"Inputs":{"shape":"OneInput"},
13858+
"PartitionKeys":{"shape":"GlueStudioPathList"},
13859+
"Path":{"shape":"EnclosedInStringProperty"},
13860+
"Format":{"shape":"TargetFormat"},
13861+
"AdditionalOptions":{"shape":"AdditionalOptions"},
13862+
"SchemaChangePolicy":{"shape":"DirectSchemaChangePolicy"},
13863+
"Compression":{"shape":"IcebergTargetCompressionType"},
13864+
"NumberTargetPartitions":{"shape":"NumberTargetPartitionsString"}
13865+
}
13866+
},
1378513867
"S3JsonSource":{
1378613868
"type":"structure",
1378713869
"required":[
@@ -15216,7 +15298,10 @@
1521615298
"orc",
1521715299
"parquet",
1521815300
"hudi",
15219-
"delta"
15301+
"delta",
15302+
"iceberg",
15303+
"hyper",
15304+
"xml"
1522015305
]
1522115306
},
1522215307
"TargetProcessingProperties":{

0 commit comments

Comments
 (0)