Skip to content

Commit e51ce9a

Browse files
author
AWS
committed
AWS Glue DataBrew Update: This SDK release adds two new features: 1) Output to Native JDBC destinations and 2) Adding configurations to profile jobs
1 parent 4f16790 commit e51ce9a

File tree

2 files changed

+194
-12
lines changed

2 files changed

+194
-12
lines changed

‎.changes/next-release/feature-AWSGlueDataBrew-d6f005a.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"type": "feature",
3+
"category": "AWS Glue DataBrew",
4+
"contributor": "",
5+
"description": "This SDK release adds two new features: 1) Output to Native JDBC destinations and 2) Adding configurations to profile jobs"
6+
}

‎services/databrew/src/main/resources/codegen-resources/service-2.json

Lines changed: 188 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -669,6 +669,45 @@
669669
"max":20,
670670
"min":0
671671
},
672+
"ColumnSelector":{
673+
"type":"structure",
674+
"members":{
675+
"Regex":{
676+
"shape":"ColumnName",
677+
"documentation":"<p>A regular expression for selecting a column from a dataset.</p>"
678+
},
679+
"Name":{
680+
"shape":"ColumnName",
681+
"documentation":"<p>The name of a column from a dataset.</p>"
682+
}
683+
},
684+
"documentation":"<p>Selector of a column from a dataset for profile job configuration. One selector includes either a column name or a regular expression.</p>"
685+
},
686+
"ColumnSelectorList":{
687+
"type":"list",
688+
"member":{"shape":"ColumnSelector"},
689+
"min":1
690+
},
691+
"ColumnStatisticsConfiguration":{
692+
"type":"structure",
693+
"required":["Statistics"],
694+
"members":{
695+
"Selectors":{
696+
"shape":"ColumnSelectorList",
697+
"documentation":"<p>List of column selectors. Selectors can be used to select columns from the dataset. When selectors are undefined, configuration will be applied to all supported columns. </p>"
698+
},
699+
"Statistics":{
700+
"shape":"StatisticsConfiguration",
701+
"documentation":"<p>Configuration for evaluations. Statistics can be used to select evaluations and override parameters of evaluations. </p>"
702+
}
703+
},
704+
"documentation":"<p>Configuration for column evaluations for a profile job. ColumnStatisticsConfiguration can be used to select evaluations and override parameters of evaluations for particular columns. </p>"
705+
},
706+
"ColumnStatisticsConfigurationList":{
707+
"type":"list",
708+
"member":{"shape":"ColumnStatisticsConfiguration"},
709+
"min":1
710+
},
672711
"CompressionFormat":{
673712
"type":"string",
674713
"enum":[
@@ -804,6 +843,10 @@
804843
"documentation":"<p>The maximum number of times to retry the job after a job run fails.</p>"
805844
},
806845
"OutputLocation":{"shape":"S3Location"},
846+
"Configuration":{
847+
"shape":"ProfileConfiguration",
848+
"documentation":"<p>Configuration for profile jobs. Used to select columns, do evaluations, and override default parameters of evaluations. When configuration is null, the profile job will run with default settings.</p>"
849+
},
807850
"RoleArn":{
808851
"shape":"Arn",
809852
"documentation":"<p>The Amazon Resource Name (ARN) of the Identity and Access Management (IAM) role to be assumed when DataBrew runs the job.</p>"
@@ -915,7 +958,11 @@
915958
},
916959
"DataCatalogOutputs":{
917960
"shape":"DataCatalogOutputList",
918-
"documentation":"<p>One or more artifacts that represent the AWS Glue Data Catalog output from running the job.</p>"
961+
"documentation":"<p>One or more artifacts that represent the Glue Data Catalog output from running the job.</p>"
962+
},
963+
"DatabaseOutputs":{
964+
"shape":"DatabaseOutputList",
965+
"documentation":"<p>Represents a list of JDBC database output objects which defines the output destination for a DataBrew recipe job to write to. </p>"
919966
},
920967
"ProjectName":{
921968
"shape":"ProjectName",
@@ -1081,7 +1128,7 @@
10811128
"members":{
10821129
"CatalogId":{
10831130
"shape":"CatalogId",
1084-
"documentation":"<p>The unique identifier of the AWS account that holds the Data Catalog that stores the data.</p>"
1131+
"documentation":"<p>The unique identifier of the Amazon Web Services account that holds the Data Catalog that stores the data.</p>"
10851132
},
10861133
"DatabaseName":{
10871134
"shape":"DatabaseName",
@@ -1093,7 +1140,7 @@
10931140
},
10941141
"S3Options":{
10951142
"shape":"S3TableOutputOptions",
1096-
"documentation":"<p>Represents options that specify how and where DataBrew writes the S3 output generated by recipe jobs.</p>"
1143+
"documentation":"<p>Represents options that specify how and where DataBrew writes the Amazon S3 output generated by recipe jobs.</p>"
10971144
},
10981145
"DatabaseOptions":{
10991146
"shape":"DatabaseTableOutputOptions",
@@ -1104,7 +1151,7 @@
11041151
"documentation":"<p>A value that, if true, means that any data in the location specified for output is overwritten with new output. Not supported with DatabaseOptions.</p>"
11051152
}
11061153
},
1107-
"documentation":"<p>Represents options that specify how and where DataBrew writes the output generated by recipe jobs.</p>"
1154+
"documentation":"<p>Represents options that specify how and where in the Glue Data Catalog DataBrew writes the output generated by recipe jobs.</p>"
11081155
},
11091156
"DataCatalogOutputList":{
11101157
"type":"list",
@@ -1135,6 +1182,37 @@
11351182
"max":255,
11361183
"min":1
11371184
},
1185+
"DatabaseOutput":{
1186+
"type":"structure",
1187+
"required":[
1188+
"GlueConnectionName",
1189+
"DatabaseOptions"
1190+
],
1191+
"members":{
1192+
"GlueConnectionName":{
1193+
"shape":"GlueConnectionName",
1194+
"documentation":"<p>The Glue connection that stores the connection information for the target database.</p>"
1195+
},
1196+
"DatabaseOptions":{
1197+
"shape":"DatabaseTableOutputOptions",
1198+
"documentation":"<p>Represents options that specify how and where DataBrew writes the database output generated by recipe jobs.</p>"
1199+
},
1200+
"DatabaseOutputMode":{
1201+
"shape":"DatabaseOutputMode",
1202+
"documentation":"<p>The output mode to write into the database. Currently supported option: NEW_TABLE.</p>"
1203+
}
1204+
},
1205+
"documentation":"<p>Represents a JDBC database output object which defines the output destination for a DataBrew recipe job to write into.</p>"
1206+
},
1207+
"DatabaseOutputList":{
1208+
"type":"list",
1209+
"member":{"shape":"DatabaseOutput"},
1210+
"min":1
1211+
},
1212+
"DatabaseOutputMode":{
1213+
"type":"string",
1214+
"enum":["NEW_TABLE"]
1215+
},
11381216
"DatabaseTableName":{
11391217
"type":"string",
11401218
"max":255,
@@ -1545,12 +1623,20 @@
15451623
},
15461624
"DataCatalogOutputs":{
15471625
"shape":"DataCatalogOutputList",
1548-
"documentation":"<p>One or more artifacts that represent the AWS Glue Data Catalog output from running the job.</p>"
1626+
"documentation":"<p>One or more artifacts that represent the Glue Data Catalog output from running the job.</p>"
1627+
},
1628+
"DatabaseOutputs":{
1629+
"shape":"DatabaseOutputList",
1630+
"documentation":"<p>Represents a list of JDBC database output objects which defines the output destination for a DataBrew recipe job to write into.</p>"
15491631
},
15501632
"ProjectName":{
15511633
"shape":"ProjectName",
15521634
"documentation":"<p>The DataBrew project associated with this job.</p>"
15531635
},
1636+
"ProfileConfiguration":{
1637+
"shape":"ProfileConfiguration",
1638+
"documentation":"<p>Configuration for profile jobs. Used to select columns, do evaluations, and override default parameters of evaluations. When configuration is null, the profile job will run with default settings.</p>"
1639+
},
15541640
"RecipeReference":{"shape":"RecipeReference"},
15551641
"ResourceArn":{
15561642
"shape":"Arn",
@@ -1623,6 +1709,10 @@
16231709
"shape":"JobName",
16241710
"documentation":"<p>The name of the job being processed during this run.</p>"
16251711
},
1712+
"ProfileConfiguration":{
1713+
"shape":"ProfileConfiguration",
1714+
"documentation":"<p>Configuration for profile jobs. Used to select columns, do evaluations, and override default parameters of evaluations. When configuration is null, the profile job will run with default settings.</p>"
1715+
},
16261716
"RunId":{
16271717
"shape":"JobRunId",
16281718
"documentation":"<p>The unique identifier of the job run.</p>"
@@ -1645,7 +1735,11 @@
16451735
},
16461736
"DataCatalogOutputs":{
16471737
"shape":"DataCatalogOutputList",
1648-
"documentation":"<p>One or more artifacts that represent the AWS Glue Data Catalog output from running the job.</p>"
1738+
"documentation":"<p>One or more artifacts that represent the Glue Data Catalog output from running the job.</p>"
1739+
},
1740+
"DatabaseOutputs":{
1741+
"shape":"DatabaseOutputList",
1742+
"documentation":"<p>Represents a list of JDBC database output objects which defines the output destination for a DataBrew recipe job to write into.</p>"
16491743
},
16501744
"RecipeReference":{"shape":"RecipeReference"},
16511745
"StartedBy":{
@@ -2067,7 +2161,11 @@
20672161
},
20682162
"DataCatalogOutputs":{
20692163
"shape":"DataCatalogOutputList",
2070-
"documentation":"<p>One or more artifacts that represent the AWS Glue Data Catalog output from running the job.</p>"
2164+
"documentation":"<p>One or more artifacts that represent the Glue Data Catalog output from running the job.</p>"
2165+
},
2166+
"DatabaseOutputs":{
2167+
"shape":"DatabaseOutputList",
2168+
"documentation":"<p>Represents a list of JDBC database output objects which defines the output destination for a DataBrew recipe job to write into.</p>"
20712169
},
20722170
"ProjectName":{
20732171
"shape":"ProjectName",
@@ -2163,7 +2261,11 @@
21632261
},
21642262
"DataCatalogOutputs":{
21652263
"shape":"DataCatalogOutputList",
2166-
"documentation":"<p>One or more artifacts that represent the AWS Glue Data Catalog output from running the job.</p>"
2264+
"documentation":"<p>One or more artifacts that represent the Glue Data Catalog output from running the job.</p>"
2265+
},
2266+
"DatabaseOutputs":{
2267+
"shape":"DatabaseOutputList",
2268+
"documentation":"<p>Represents a list of JDBC database output objects which defines the output destination for a DataBrew recipe job to write into.</p>"
21672269
},
21682270
"RecipeReference":{
21692271
"shape":"RecipeReference",
@@ -2606,7 +2708,7 @@
26062708
"documentation":"<p>Represents options that define how DataBrew formats job output files.</p>"
26072709
}
26082710
},
2609-
"documentation":"<p>Represents options that specify how and where DataBrew writes the output generated by recipe jobs or profile jobs.</p>"
2711+
"documentation":"<p>Represents options that specify how and where in Amazon S3 DataBrew writes the output generated by recipe jobs or profile jobs.</p>"
26102712
},
26112713
"OutputFormat":{
26122714
"type":"string",
@@ -2657,7 +2759,7 @@
26572759
},
26582760
"ParameterValue":{
26592761
"type":"string",
2660-
"max":12288,
2762+
"max":32768,
26612763
"min":1
26622764
},
26632765
"PathOptions":{
@@ -2692,6 +2794,24 @@
26922794
"min":1
26932795
},
26942796
"Preview":{"type":"boolean"},
2797+
"ProfileConfiguration":{
2798+
"type":"structure",
2799+
"members":{
2800+
"DatasetStatisticsConfiguration":{
2801+
"shape":"StatisticsConfiguration",
2802+
"documentation":"<p>Configuration for inter-column evaluations. Configuration can be used to select evaluations and override parameters of evaluations. When configuration is undefined, the profile job will run all supported inter-column evaluations. </p>"
2803+
},
2804+
"ProfileColumns":{
2805+
"shape":"ColumnSelectorList",
2806+
"documentation":"<p>List of column selectors. ProfileColumns can be used to select columns from the dataset. When ProfileColumns is undefined, the profile job will profile all supported columns. </p>"
2807+
},
2808+
"ColumnStatisticsConfigurations":{
2809+
"shape":"ColumnStatisticsConfigurationList",
2810+
"documentation":"<p>List of configurations for column evaluations. ColumnStatisticsConfigurations are used to select evaluations and override parameters of evaluations for particular columns. When ColumnStatisticsConfigurations is undefined, the profile job will profile all supported columns and run all supported evaluations. </p>"
2811+
}
2812+
},
2813+
"documentation":"<p>Configuration for profile jobs. Configuration can be used to select columns, do evaluations, and override default parameters of evaluations. When configuration is undefined, the profile job will apply default settings to all supported columns. </p>"
2814+
},
26952815
"Project":{
26962816
"type":"structure",
26972817
"required":[
@@ -2983,7 +3103,7 @@
29833103
"documentation":"<p>Represents an Amazon S3 location (bucket name and object key) where DataBrew can write output from a job.</p>"
29843104
}
29853105
},
2986-
"documentation":"<p>Represents options that specify how and where DataBrew writes the S3 output generated by recipe jobs.</p>"
3106+
"documentation":"<p>Represents options that specify how and where DataBrew writes the Amazon S3 output generated by recipe jobs.</p>"
29873107
},
29883108
"Sample":{
29893109
"type":"structure",
@@ -3231,6 +3351,54 @@
32313351
}
32323352
},
32333353
"StartedBy":{"type":"string"},
3354+
"Statistic":{
3355+
"type":"string",
3356+
"max":128,
3357+
"min":1,
3358+
"pattern":"^[A-Z\\_]+$"
3359+
},
3360+
"StatisticList":{
3361+
"type":"list",
3362+
"member":{"shape":"Statistic"},
3363+
"min":1
3364+
},
3365+
"StatisticOverride":{
3366+
"type":"structure",
3367+
"required":[
3368+
"Statistic",
3369+
"Parameters"
3370+
],
3371+
"members":{
3372+
"Statistic":{
3373+
"shape":"Statistic",
3374+
"documentation":"<p>The name of an evaluation</p>"
3375+
},
3376+
"Parameters":{
3377+
"shape":"ParameterMap",
3378+
"documentation":"<p>A map that includes overrides of an evaluation’s parameters.</p>"
3379+
}
3380+
},
3381+
"documentation":"<p>Override of a particular evaluation for a profile job. </p>"
3382+
},
3383+
"StatisticOverrideList":{
3384+
"type":"list",
3385+
"member":{"shape":"StatisticOverride"},
3386+
"min":1
3387+
},
3388+
"StatisticsConfiguration":{
3389+
"type":"structure",
3390+
"members":{
3391+
"IncludedStatistics":{
3392+
"shape":"StatisticList",
3393+
"documentation":"<p>List of included evaluations. When the list is undefined, all supported evaluations will be included.</p>"
3394+
},
3395+
"Overrides":{
3396+
"shape":"StatisticOverrideList",
3397+
"documentation":"<p>List of overrides for evaluations.</p>"
3398+
}
3399+
},
3400+
"documentation":"<p>Configuration of evaluations for a profile job. This configuration can be used to select evaluations and override the parameters of selected evaluations. </p>"
3401+
},
32343402
"StepIndex":{
32353403
"type":"integer",
32363404
"min":0
@@ -3401,6 +3569,10 @@
34013569
"RoleArn"
34023570
],
34033571
"members":{
3572+
"Configuration":{
3573+
"shape":"ProfileConfiguration",
3574+
"documentation":"<p>Configuration for profile jobs. Used to select columns, do evaluations, and override default parameters of evaluations. When configuration is null, the profile job will run with default settings.</p>"
3575+
},
34043576
"EncryptionKeyArn":{
34053577
"shape":"EncryptionKeyArn",
34063578
"documentation":"<p>The Amazon Resource Name (ARN) of an encryption key that is used to protect the job.</p>"
@@ -3525,7 +3697,11 @@
35253697
},
35263698
"DataCatalogOutputs":{
35273699
"shape":"DataCatalogOutputList",
3528-
"documentation":"<p>One or more artifacts that represent the AWS Glue Data Catalog output from running the job.</p>"
3700+
"documentation":"<p>One or more artifacts that represent the Glue Data Catalog output from running the job.</p>"
3701+
},
3702+
"DatabaseOutputs":{
3703+
"shape":"DatabaseOutputList",
3704+
"documentation":"<p>Represents a list of JDBC database output objects which defines the output destination for a DataBrew recipe job to write into.</p>"
35293705
},
35303706
"RoleArn":{
35313707
"shape":"Arn",

0 commit comments

Comments
 (0)