Skip to content

Commit 5004266

Browse files
author
AWS
committed
EMR Serverless Update: The release adds support for spark structured streaming.
1 parent 00de16a commit 5004266

File tree

3 files changed

+258
-1
lines changed

3 files changed

+258
-1
lines changed

‎.changes/next-release/feature-EMRServerless-3e00816.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"type": "feature",
3+
"category": "EMR Serverless",
4+
"contributor": "",
5+
"description": "The release adds support for spark structured streaming."
6+
}

‎services/emrserverless/src/main/resources/codegen-resources/paginators-1.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,12 @@
66
"limit_key": "maxResults",
77
"result_key": "applications"
88
},
9+
"ListJobRunAttempts": {
10+
"input_token": "nextToken",
11+
"output_token": "nextToken",
12+
"limit_key": "maxResults",
13+
"result_key": "jobRunAttempts"
14+
},
915
"ListJobRuns": {
1016
"input_token": "nextToken",
1117
"output_token": "nextToken",

‎services/emrserverless/src/main/resources/codegen-resources/service-2.json

Lines changed: 246 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
"metadata":{
44
"apiVersion":"2021-07-13",
55
"endpointPrefix":"emr-serverless",
6+
"jsonVersion":"1.1",
67
"protocol":"rest-json",
7-
"protocols":["rest-json"],
88
"serviceFullName":"EMR Serverless",
99
"serviceId":"EMR Serverless",
1010
"signatureVersion":"v4",
@@ -127,6 +127,22 @@
127127
],
128128
"documentation":"<p>Lists applications based on a set of parameters.</p>"
129129
},
130+
"ListJobRunAttempts":{
131+
"name":"ListJobRunAttempts",
132+
"http":{
133+
"method":"GET",
134+
"requestUri":"/applications/{applicationId}/jobruns/{jobRunId}/attempts",
135+
"responseCode":200
136+
},
137+
"input":{"shape":"ListJobRunAttemptsRequest"},
138+
"output":{"shape":"ListJobRunAttemptsResponse"},
139+
"errors":[
140+
{"shape":"ValidationException"},
141+
{"shape":"ResourceNotFoundException"},
142+
{"shape":"InternalServerException"}
143+
],
144+
"documentation":"<p>Lists all attempt of a job run.</p>"
145+
},
130146
"ListJobRuns":{
131147
"name":"ListJobRuns",
132148
"http":{
@@ -460,6 +476,11 @@
460476
"X86_64"
461477
]
462478
},
479+
"AttemptNumber":{
480+
"type":"integer",
481+
"box":true,
482+
"min":1
483+
},
463484
"AutoStartConfig":{
464485
"type":"structure",
465486
"members":{
@@ -837,6 +858,12 @@
837858
"documentation":"<p>The ID of the job run.</p>",
838859
"location":"uri",
839860
"locationName":"jobRunId"
861+
},
862+
"attempt":{
863+
"shape":"AttemptNumber",
864+
"documentation":"<p>An optimal parameter that indicates the amount of attempts for the job. If not specified, this value defaults to the attempt of the latest job.</p>",
865+
"location":"querystring",
866+
"locationName":"attempt"
840867
}
841868
}
842869
},
@@ -867,6 +894,12 @@
867894
"documentation":"<p>The ID of the job run.</p>",
868895
"location":"uri",
869896
"locationName":"jobRunId"
897+
},
898+
"attempt":{
899+
"shape":"AttemptNumber",
900+
"documentation":"<p>An optimal parameter that indicates the amount of attempts for the job. If not specified, this value defaults to the attempt of the latest job.</p>",
901+
"location":"querystring",
902+
"locationName":"attempt"
870903
}
871904
}
872905
},
@@ -1115,16 +1148,126 @@
11151148
"billedResourceUtilization":{
11161149
"shape":"ResourceUtilization",
11171150
"documentation":"<p>The aggregate vCPU, memory, and storage that Amazon Web Services has billed for the job run. The billed resources include a 1-minute minimum usage for workers, plus additional storage over 20 GB per worker. Note that billed resources do not include usage for idle pre-initialized workers.</p>"
1151+
},
1152+
"mode":{
1153+
"shape":"JobRunMode",
1154+
"documentation":"<p>The mode of the job run.</p>"
1155+
},
1156+
"retryPolicy":{
1157+
"shape":"RetryPolicy",
1158+
"documentation":"<p>The retry policy of the job run.</p>"
1159+
},
1160+
"attempt":{
1161+
"shape":"AttemptNumber",
1162+
"documentation":"<p>The attempt of the job run.</p>"
1163+
},
1164+
"attemptCreatedAt":{
1165+
"shape":"Date",
1166+
"documentation":"<p>The date and time of when the job run attempt was created.</p>"
1167+
},
1168+
"attemptUpdatedAt":{
1169+
"shape":"Date",
1170+
"documentation":"<p>The date and time of when the job run attempt was last updated.</p>"
11181171
}
11191172
},
11201173
"documentation":"<p>Information about a job run. A job run is a unit of work, such as a Spark JAR, Hive query, or SparkSQL query, that you submit to an Amazon EMR Serverless application.</p>"
11211174
},
1175+
"JobRunAttemptSummary":{
1176+
"type":"structure",
1177+
"required":[
1178+
"applicationId",
1179+
"id",
1180+
"arn",
1181+
"createdBy",
1182+
"jobCreatedAt",
1183+
"createdAt",
1184+
"updatedAt",
1185+
"executionRole",
1186+
"state",
1187+
"stateDetails",
1188+
"releaseLabel"
1189+
],
1190+
"members":{
1191+
"applicationId":{
1192+
"shape":"ApplicationId",
1193+
"documentation":"<p>The ID of the application the job is running on.</p>"
1194+
},
1195+
"id":{
1196+
"shape":"JobRunId",
1197+
"documentation":"<p>The ID of the job run attempt.</p>"
1198+
},
1199+
"name":{
1200+
"shape":"String256",
1201+
"documentation":"<p>The name of the job run attempt.</p>"
1202+
},
1203+
"mode":{
1204+
"shape":"JobRunMode",
1205+
"documentation":"<p>The mode of the job run attempt.</p>"
1206+
},
1207+
"arn":{
1208+
"shape":"JobArn",
1209+
"documentation":"<p>The Amazon Resource Name (ARN) of the job run.</p>"
1210+
},
1211+
"createdBy":{
1212+
"shape":"RequestIdentityUserArn",
1213+
"documentation":"<p>The user who created the job run.</p>"
1214+
},
1215+
"jobCreatedAt":{
1216+
"shape":"Date",
1217+
"documentation":"<p>The date and time of when the job run was created.</p>"
1218+
},
1219+
"createdAt":{
1220+
"shape":"Date",
1221+
"documentation":"<p>The date and time when the job run attempt was created.</p>"
1222+
},
1223+
"updatedAt":{
1224+
"shape":"Date",
1225+
"documentation":"<p>The date and time of when the job run attempt was last updated.</p>"
1226+
},
1227+
"executionRole":{
1228+
"shape":"IAMRoleArn",
1229+
"documentation":"<p>The Amazon Resource Name (ARN) of the execution role of the job run..</p>"
1230+
},
1231+
"state":{
1232+
"shape":"JobRunState",
1233+
"documentation":"<p>The state of the job run attempt.</p>"
1234+
},
1235+
"stateDetails":{
1236+
"shape":"String256",
1237+
"documentation":"<p>The state details of the job run attempt.</p>"
1238+
},
1239+
"releaseLabel":{
1240+
"shape":"ReleaseLabel",
1241+
"documentation":"<p>The Amazon EMR release label of the job run attempt.</p>"
1242+
},
1243+
"type":{
1244+
"shape":"JobRunType",
1245+
"documentation":"<p>The type of the job run, such as Spark or Hive.</p>"
1246+
},
1247+
"attempt":{
1248+
"shape":"AttemptNumber",
1249+
"documentation":"<p>The attempt number of the job run execution.</p>"
1250+
}
1251+
},
1252+
"documentation":"<p>The summary of attributes associated with a job run attempt.</p>"
1253+
},
1254+
"JobRunAttempts":{
1255+
"type":"list",
1256+
"member":{"shape":"JobRunAttemptSummary"}
1257+
},
11221258
"JobRunId":{
11231259
"type":"string",
11241260
"max":64,
11251261
"min":1,
11261262
"pattern":"[0-9a-z]+"
11271263
},
1264+
"JobRunMode":{
1265+
"type":"string",
1266+
"enum":[
1267+
"BATCH",
1268+
"STREAMING"
1269+
]
1270+
},
11281271
"JobRunState":{
11291272
"type":"string",
11301273
"enum":[
@@ -1171,6 +1314,10 @@
11711314
"shape":"String256",
11721315
"documentation":"<p>The optional job run name. This doesn't have to be unique.</p>"
11731316
},
1317+
"mode":{
1318+
"shape":"JobRunMode",
1319+
"documentation":"<p>The mode of the job run.</p>"
1320+
},
11741321
"arn":{
11751322
"shape":"JobArn",
11761323
"documentation":"<p>The ARN of the job run.</p>"
@@ -1206,6 +1353,18 @@
12061353
"type":{
12071354
"shape":"JobRunType",
12081355
"documentation":"<p>The type of job run, such as Spark or Hive.</p>"
1356+
},
1357+
"attempt":{
1358+
"shape":"AttemptNumber",
1359+
"documentation":"<p>The attempt number of the job run execution.</p>"
1360+
},
1361+
"attemptCreatedAt":{
1362+
"shape":"Date",
1363+
"documentation":"<p>The date and time of when the job run attempt was created.</p>"
1364+
},
1365+
"attemptUpdatedAt":{
1366+
"shape":"Date",
1367+
"documentation":"<p>The date and time of when the job run attempt was last updated.</p>"
12091368
}
12101369
},
12111370
"documentation":"<p>The summary of attributes associated with a job run.</p>"
@@ -1258,6 +1417,59 @@
12581417
}
12591418
}
12601419
},
1420+
"ListJobRunAttemptsRequest":{
1421+
"type":"structure",
1422+
"required":[
1423+
"applicationId",
1424+
"jobRunId"
1425+
],
1426+
"members":{
1427+
"applicationId":{
1428+
"shape":"ApplicationId",
1429+
"documentation":"<p>The ID of the application for which to list job runs.</p>",
1430+
"location":"uri",
1431+
"locationName":"applicationId"
1432+
},
1433+
"jobRunId":{
1434+
"shape":"JobRunId",
1435+
"documentation":"<p>The ID of the job run to list.</p>",
1436+
"location":"uri",
1437+
"locationName":"jobRunId"
1438+
},
1439+
"nextToken":{
1440+
"shape":"NextToken",
1441+
"documentation":"<p>The token for the next set of job run attempt results.</p>",
1442+
"location":"querystring",
1443+
"locationName":"nextToken"
1444+
},
1445+
"maxResults":{
1446+
"shape":"ListJobRunAttemptsRequestMaxResultsInteger",
1447+
"documentation":"<p>The maximum number of job run attempts to list.</p>",
1448+
"location":"querystring",
1449+
"locationName":"maxResults"
1450+
}
1451+
}
1452+
},
1453+
"ListJobRunAttemptsRequestMaxResultsInteger":{
1454+
"type":"integer",
1455+
"box":true,
1456+
"max":50,
1457+
"min":1
1458+
},
1459+
"ListJobRunAttemptsResponse":{
1460+
"type":"structure",
1461+
"required":["jobRunAttempts"],
1462+
"members":{
1463+
"jobRunAttempts":{
1464+
"shape":"JobRunAttempts",
1465+
"documentation":"<p>The array of the listed job run attempt objects.</p>"
1466+
},
1467+
"nextToken":{
1468+
"shape":"NextToken",
1469+
"documentation":"<p>The output displays the token for the next set of application results. This is required for pagination and is available as a response of the previous request.</p>"
1470+
}
1471+
}
1472+
},
12611473
"ListJobRunsRequest":{
12621474
"type":"structure",
12631475
"required":["applicationId"],
@@ -1297,6 +1509,12 @@
12971509
"documentation":"<p>An optional filter for job run states. Note that if this filter contains multiple states, the resulting list will be grouped by the state.</p>",
12981510
"location":"querystring",
12991511
"locationName":"states"
1512+
},
1513+
"mode":{
1514+
"shape":"JobRunMode",
1515+
"documentation":"<p>The mode of the job runs to list.</p>",
1516+
"location":"querystring",
1517+
"locationName":"mode"
13001518
}
13011519
}
13021520
},
@@ -1529,6 +1747,25 @@
15291747
},
15301748
"documentation":"<p>The resource utilization for memory, storage, and vCPU for jobs.</p>"
15311749
},
1750+
"RetryPolicy":{
1751+
"type":"structure",
1752+
"members":{
1753+
"maxAttempts":{
1754+
"shape":"AttemptNumber",
1755+
"documentation":"<p>Maximum number of attempts for the job run. This parameter is only applicable for <code>BATCH</code> mode.</p>"
1756+
},
1757+
"maxFailedAttemptsPerHour":{
1758+
"shape":"RetryPolicyMaxFailedAttemptsPerHourInteger",
1759+
"documentation":"<p>Maximum number of failed attempts per hour. This [arameter is only applicable for <code>STREAMING</code> mode.</p>"
1760+
}
1761+
},
1762+
"documentation":"<p>The retry policy to use for a job run.</p>"
1763+
},
1764+
"RetryPolicyMaxFailedAttemptsPerHourInteger":{
1765+
"type":"integer",
1766+
"box":true,
1767+
"min":1
1768+
},
15321769
"S3MonitoringConfiguration":{
15331770
"type":"structure",
15341771
"members":{
@@ -1662,6 +1899,14 @@
16621899
"name":{
16631900
"shape":"String256",
16641901
"documentation":"<p>The optional job run name. This doesn't have to be unique.</p>"
1902+
},
1903+
"mode":{
1904+
"shape":"JobRunMode",
1905+
"documentation":"<p>The mode of the job run when it starts.</p>"
1906+
},
1907+
"retryPolicy":{
1908+
"shape":"RetryPolicy",
1909+
"documentation":"<p>The retry policy when job run starts.</p>"
16651910
}
16661911
}
16671912
},

0 commit comments

Comments
 (0)