Skip to content

Commit 76f9fe2

Browse files
aws-sdk-dotnet-automationaws-sdk-dotnet-automation
committed
AWS Glue Crawlers can now accept SerDe overrides from a custom csv classifier. The two SerDe options are LazySimpleSerDe and OpenCSVSerDe. In case, the user wants crawler to do the selection, "None" can be selected for this purpose.
1 parent 3256d09 commit 76f9fe2

File tree

10 files changed

+180
-3
lines changed

10 files changed

+180
-3
lines changed

‎generator/ServiceModels/glue/glue-2017-03-31.api.json

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4933,7 +4933,8 @@
49334933
"DisableValueTrimming":{"shape":"NullableBoolean"},
49344934
"AllowSingleColumn":{"shape":"NullableBoolean"},
49354935
"CustomDatatypeConfigured":{"shape":"NullableBoolean"},
4936-
"CustomDatatypes":{"shape":"CustomDatatypes"}
4936+
"CustomDatatypes":{"shape":"CustomDatatypes"},
4937+
"Serde":{"shape":"CsvSerdeOption"}
49374938
}
49384939
},
49394940
"CreateCustomEntityTypeRequest":{
@@ -5399,7 +5400,8 @@
53995400
"DisableValueTrimming":{"shape":"NullableBoolean"},
54005401
"AllowSingleColumn":{"shape":"NullableBoolean"},
54015402
"CustomDatatypeConfigured":{"shape":"NullableBoolean"},
5402-
"CustomDatatypes":{"shape":"CustomDatatypes"}
5403+
"CustomDatatypes":{"shape":"CustomDatatypes"},
5404+
"Serde":{"shape":"CsvSerdeOption"}
54035405
}
54045406
},
54055407
"CsvColumnDelimiter":{
@@ -5426,6 +5428,14 @@
54265428
"min":1,
54275429
"pattern":"[^\\r\\n]"
54285430
},
5431+
"CsvSerdeOption":{
5432+
"type":"string",
5433+
"enum":[
5434+
"OpenCSVSerDe",
5435+
"LazySimpleSerDe",
5436+
"None"
5437+
]
5438+
},
54295439
"CustomCode":{
54305440
"type":"structure",
54315441
"required":[
@@ -12244,7 +12254,8 @@
1224412254
"DisableValueTrimming":{"shape":"NullableBoolean"},
1224512255
"AllowSingleColumn":{"shape":"NullableBoolean"},
1224612256
"CustomDatatypeConfigured":{"shape":"NullableBoolean"},
12247-
"CustomDatatypes":{"shape":"CustomDatatypes"}
12257+
"CustomDatatypes":{"shape":"CustomDatatypes"},
12258+
"Serde":{"shape":"CsvSerdeOption"}
1224812259
}
1224912260
},
1225012261
"UpdateDataQualityRulesetRequest":{

‎generator/ServiceModels/glue/glue-2017-03-31.docs.json

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1901,6 +1901,14 @@
19011901
"UpdateCsvClassifierRequest$QuoteSymbol": "<p>A custom symbol to denote what combines content into a single column value. It must be different from the column delimiter.</p>"
19021902
}
19031903
},
1904+
"CsvSerdeOption": {
1905+
"base": null,
1906+
"refs": {
1907+
"CreateCsvClassifierRequest$Serde": "<p>Sets the SerDe for processing CSV in the classifier, which will be applied in the Data Catalog. Valid values are <code>OpenCSVSerDe</code>, <code>LazySimpleSerDe</code>, and <code>None</code>. You can specify the <code>None</code> value when you want the crawler to do the detection.</p>",
1908+
"CsvClassifier$Serde": "<p>Sets the SerDe for processing CSV in the classifier, which will be applied in the Data Catalog. Valid values are <code>OpenCSVSerDe</code>, <code>LazySimpleSerDe</code>, and <code>None</code>. You can specify the <code>None</code> value when you want the crawler to do the detection.</p>",
1909+
"UpdateCsvClassifierRequest$Serde": "<p>Sets the SerDe for processing CSV in the classifier, which will be applied in the Data Catalog. Valid values are <code>OpenCSVSerDe</code>, <code>LazySimpleSerDe</code>, and <code>None</code>. You can specify the <code>None</code> value when you want the crawler to do the detection.</p>"
1910+
}
1911+
},
19041912
"CustomCode": {
19051913
"base": "<p>Specifies a transform that uses custom code you provide to perform the data transformation. The output is a collection of DynamicFrames.</p>",
19061914
"refs": {

‎generator/ServiceModels/glue/glue-2017-03-31.normal.json

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6562,6 +6562,10 @@
65626562
"CustomDatatypes":{
65636563
"shape":"CustomDatatypes",
65646564
"documentation":"<p>Creates a list of supported custom datatypes.</p>"
6565+
},
6566+
"Serde":{
6567+
"shape":"CsvSerdeOption",
6568+
"documentation":"<p>Sets the SerDe for processing CSV in the classifier, which will be applied in the Data Catalog. Valid values are <code>OpenCSVSerDe</code>, <code>LazySimpleSerDe</code>, and <code>None</code>. You can specify the <code>None</code> value when you want the crawler to do the detection.</p>"
65656569
}
65666570
},
65676571
"documentation":"<p>Specifies a custom CSV classifier for <code>CreateClassifier</code> to create.</p>"
@@ -7613,6 +7617,10 @@
76137617
"CustomDatatypes":{
76147618
"shape":"CustomDatatypes",
76157619
"documentation":"<p>A list of custom datatypes including \"BINARY\", \"BOOLEAN\", \"DATE\", \"DECIMAL\", \"DOUBLE\", \"FLOAT\", \"INT\", \"LONG\", \"SHORT\", \"STRING\", \"TIMESTAMP\".</p>"
7620+
},
7621+
"Serde":{
7622+
"shape":"CsvSerdeOption",
7623+
"documentation":"<p>Sets the SerDe for processing CSV in the classifier, which will be applied in the Data Catalog. Valid values are <code>OpenCSVSerDe</code>, <code>LazySimpleSerDe</code>, and <code>None</code>. You can specify the <code>None</code> value when you want the crawler to do the detection.</p>"
76167624
}
76177625
},
76187626
"documentation":"<p>A classifier for custom <code>CSV</code> content.</p>"
@@ -7641,6 +7649,14 @@
76417649
"min":1,
76427650
"pattern":"[^\\r\\n]"
76437651
},
7652+
"CsvSerdeOption":{
7653+
"type":"string",
7654+
"enum":[
7655+
"OpenCSVSerDe",
7656+
"LazySimpleSerDe",
7657+
"None"
7658+
]
7659+
},
76447660
"CustomCode":{
76457661
"type":"structure",
76467662
"required":[
@@ -20074,6 +20090,10 @@
2007420090
"CustomDatatypes":{
2007520091
"shape":"CustomDatatypes",
2007620092
"documentation":"<p>Specifies a list of supported custom datatypes.</p>"
20093+
},
20094+
"Serde":{
20095+
"shape":"CsvSerdeOption",
20096+
"documentation":"<p>Sets the SerDe for processing CSV in the classifier, which will be applied in the Data Catalog. Valid values are <code>OpenCSVSerDe</code>, <code>LazySimpleSerDe</code>, and <code>None</code>. You can specify the <code>None</code> value when you want the crawler to do the detection.</p>"
2007720097
}
2007820098
},
2007920099
"documentation":"<p>Specifies a custom CSV classifier to be updated.</p>"

‎sdk/src/Services/Glue/Generated/Model/CreateCsvClassifierRequest.cs

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ public partial class CreateCsvClassifierRequest
4242
private List<string> _header = new List<string>();
4343
private string _name;
4444
private string _quoteSymbol;
45+
private CsvSerdeOption _serde;
4546

4647
/// <summary>
4748
/// Gets and sets the property AllowSingleColumn.
@@ -210,5 +211,26 @@ internal bool IsSetQuoteSymbol()
210211
return this._quoteSymbol != null;
211212
}
212213

214+
/// <summary>
215+
/// Gets and sets the property Serde.
216+
/// <para>
217+
/// Sets the SerDe for processing CSV in the classifier, which will be applied in the
218+
/// Data Catalog. Valid values are <code>OpenCSVSerDe</code>, <code>LazySimpleSerDe</code>,
219+
/// and <code>None</code>. You can specify the <code>None</code> value when you want the
220+
/// crawler to do the detection.
221+
/// </para>
222+
/// </summary>
223+
public CsvSerdeOption Serde
224+
{
225+
get { return this._serde; }
226+
set { this._serde = value; }
227+
}
228+
229+
// Check to see if Serde property is set
230+
internal bool IsSetSerde()
231+
{
232+
return this._serde != null;
233+
}
234+
213235
}
214236
}

‎sdk/src/Services/Glue/Generated/Model/CsvClassifier.cs

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ public partial class CsvClassifier
4444
private DateTime? _lastUpdated;
4545
private string _name;
4646
private string _quoteSymbol;
47+
private CsvSerdeOption _serde;
4748
private long? _version;
4849

4950
/// <summary>
@@ -250,6 +251,27 @@ internal bool IsSetQuoteSymbol()
250251
return this._quoteSymbol != null;
251252
}
252253

254+
/// <summary>
255+
/// Gets and sets the property Serde.
256+
/// <para>
257+
/// Sets the SerDe for processing CSV in the classifier, which will be applied in the
258+
/// Data Catalog. Valid values are <code>OpenCSVSerDe</code>, <code>LazySimpleSerDe</code>,
259+
/// and <code>None</code>. You can specify the <code>None</code> value when you want the
260+
/// crawler to do the detection.
261+
/// </para>
262+
/// </summary>
263+
public CsvSerdeOption Serde
264+
{
265+
get { return this._serde; }
266+
set { this._serde = value; }
267+
}
268+
269+
// Check to see if Serde property is set
270+
internal bool IsSetSerde()
271+
{
272+
return this._serde != null;
273+
}
274+
253275
/// <summary>
254276
/// Gets and sets the property Version.
255277
/// <para>

‎sdk/src/Services/Glue/Generated/Model/Internal/MarshallTransformations/CreateCsvClassifierRequestMarshaller.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,12 @@ public void Marshall(CreateCsvClassifierRequest requestObject, JsonMarshallerCon
109109
context.Writer.Write(requestObject.QuoteSymbol);
110110
}
111111

112+
if(requestObject.IsSetSerde())
113+
{
114+
context.Writer.WritePropertyName("Serde");
115+
context.Writer.Write(requestObject.Serde);
116+
}
117+
112118
}
113119

114120
/// <summary>

‎sdk/src/Services/Glue/Generated/Model/Internal/MarshallTransformations/CsvClassifierUnmarshaller.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,12 @@ public CsvClassifier Unmarshall(JsonUnmarshallerContext context)
130130
unmarshalledObject.QuoteSymbol = unmarshaller.Unmarshall(context);
131131
continue;
132132
}
133+
if (context.TestExpression("Serde", targetDepth))
134+
{
135+
var unmarshaller = StringUnmarshaller.Instance;
136+
unmarshalledObject.Serde = unmarshaller.Unmarshall(context);
137+
continue;
138+
}
133139
if (context.TestExpression("Version", targetDepth))
134140
{
135141
var unmarshaller = LongUnmarshaller.Instance;

‎sdk/src/Services/Glue/Generated/Model/Internal/MarshallTransformations/UpdateCsvClassifierRequestMarshaller.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,12 @@ public void Marshall(UpdateCsvClassifierRequest requestObject, JsonMarshallerCon
109109
context.Writer.Write(requestObject.QuoteSymbol);
110110
}
111111

112+
if(requestObject.IsSetSerde())
113+
{
114+
context.Writer.WritePropertyName("Serde");
115+
context.Writer.Write(requestObject.Serde);
116+
}
117+
112118
}
113119

114120
/// <summary>

‎sdk/src/Services/Glue/Generated/Model/UpdateCsvClassifierRequest.cs

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ public partial class UpdateCsvClassifierRequest
4242
private List<string> _header = new List<string>();
4343
private string _name;
4444
private string _quoteSymbol;
45+
private CsvSerdeOption _serde;
4546

4647
/// <summary>
4748
/// Gets and sets the property AllowSingleColumn.
@@ -210,5 +211,26 @@ internal bool IsSetQuoteSymbol()
210211
return this._quoteSymbol != null;
211212
}
212213

214+
/// <summary>
215+
/// Gets and sets the property Serde.
216+
/// <para>
217+
/// Sets the SerDe for processing CSV in the classifier, which will be applied in the
218+
/// Data Catalog. Valid values are <code>OpenCSVSerDe</code>, <code>LazySimpleSerDe</code>,
219+
/// and <code>None</code>. You can specify the <code>None</code> value when you want the
220+
/// crawler to do the detection.
221+
/// </para>
222+
/// </summary>
223+
public CsvSerdeOption Serde
224+
{
225+
get { return this._serde; }
226+
set { this._serde = value; }
227+
}
228+
229+
// Check to see if Serde property is set
230+
internal bool IsSetSerde()
231+
{
232+
return this._serde != null;
233+
}
234+
213235
}
214236
}

‎sdk/src/Services/Glue/Generated/ServiceEnumerations.cs

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1256,6 +1256,60 @@ public static implicit operator CsvHeaderOption(string value)
12561256
}
12571257

12581258

1259+
/// <summary>
1260+
/// Constants used for properties of type CsvSerdeOption.
1261+
/// </summary>
1262+
public class CsvSerdeOption : ConstantClass
1263+
{
1264+
1265+
/// <summary>
1266+
/// Constant LazySimpleSerDe for CsvSerdeOption
1267+
/// </summary>
1268+
public static readonly CsvSerdeOption LazySimpleSerDe = new CsvSerdeOption("LazySimpleSerDe");
1269+
/// <summary>
1270+
/// Constant None for CsvSerdeOption
1271+
/// </summary>
1272+
public static readonly CsvSerdeOption None = new CsvSerdeOption("None");
1273+
/// <summary>
1274+
/// Constant OpenCSVSerDe for CsvSerdeOption
1275+
/// </summary>
1276+
public static readonly CsvSerdeOption OpenCSVSerDe = new CsvSerdeOption("OpenCSVSerDe");
1277+
1278+
/// <summary>
1279+
/// This constant constructor does not need to be called if the constant
1280+
/// you are attempting to use is already defined as a static instance of
1281+
/// this class.
1282+
/// This constructor should be used to construct constants that are not
1283+
/// defined as statics, for instance if attempting to use a feature that is
1284+
/// newer than the current version of the SDK.
1285+
/// </summary>
1286+
public CsvSerdeOption(string value)
1287+
: base(value)
1288+
{
1289+
}
1290+
1291+
/// <summary>
1292+
/// Finds the constant for the unique value.
1293+
/// </summary>
1294+
/// <param name="value">The unique value for the constant</param>
1295+
/// <returns>The constant for the unique value</returns>
1296+
public static CsvSerdeOption FindValue(string value)
1297+
{
1298+
return FindValue<CsvSerdeOption>(value);
1299+
}
1300+
1301+
/// <summary>
1302+
/// Utility method to convert strings to the constant class.
1303+
/// </summary>
1304+
/// <param name="value">The string value to convert to the constant class.</param>
1305+
/// <returns></returns>
1306+
public static implicit operator CsvSerdeOption(string value)
1307+
{
1308+
return FindValue(value);
1309+
}
1310+
}
1311+
1312+
12591313
/// <summary>
12601314
/// Constants used for properties of type DataFormat.
12611315
/// </summary>

0 commit comments

Comments
 (0)