This release supports additional ConversionSpec parameter as part of IntegrationPartition Structure in CreateIntegrationTableProperty API. This parameter is referred to apply appropriate column transformation for columns that are used for timestamp based partitioning

Author: aws-sdk-dotnet-automation

generator/ServiceModels/glue/glue-2017-03-31.api.json

@@ -10868,7 +10868,8 @@
       "type":"structure",
       "members":{
         "FieldName":{"shape":"String128"},
-        "FunctionSpec":{"shape":"String128"}
+        "FunctionSpec":{"shape":"String128"},
+        "ConversionSpec":{"shape":"String128"}
       }
     },
     "IntegrationPartitionSpecList":{

generator/ServiceModels/glue/glue-2017-03-31.docs.json

@@ -9513,7 +9513,7 @@
         "CreateIntegrationResponse$TargetArn": "<p>The ARN of the target resource for the integration.</p>",
         "CreateIntegrationResponse$IntegrationName": "<p>A unique name for an integration in Glue.</p>",
         "CreateIntegrationResponse$IntegrationArn": "<p>The Amazon Resource Name (ARN) for the created integration.</p>",
-        "CreateIntegrationTablePropertiesRequest$ResourceArn": "<p>The connection ARN of the source, or the database ARN of the target.</p>",
+        "CreateIntegrationTablePropertiesRequest$ResourceArn": "<p>The Amazon Resource Name (ARN) of the target table for which to create integration table properties. Currently, this API only supports creating integration table properties for target tables, and the provided ARN should be the ARN of the target table in the Glue Data Catalog. Support for creating integration table properties for source connections (using the connection ARN) is not yet implemented and will be added in a future release. </p>",
         "CreateIntegrationTablePropertiesRequest$TableName": "<p>The name of the table to be replicated.</p>",
         "DeleteIntegrationRequest$IntegrationIdentifier": "<p>The Amazon Resource Name (ARN) for the integration.</p>",
         "DeleteIntegrationResponse$SourceArn": "<p>The ARN of the source for the integration.</p>",
@@ -9531,9 +9531,9 @@
         "DescribeIntegrationsResponse$Marker": "<p>A value that indicates the starting point for the next set of response records in a subsequent request.</p>",
         "GetIntegrationResourcePropertyRequest$ResourceArn": "<p>The connection ARN of the source, or the database ARN of the target.</p>",
         "GetIntegrationResourcePropertyResponse$ResourceArn": "<p>The connection ARN of the source, or the database ARN of the target.</p>",
-        "GetIntegrationTablePropertiesRequest$ResourceArn": "<p>The connection ARN of the source, or the database ARN of the target.</p>",
+        "GetIntegrationTablePropertiesRequest$ResourceArn": "<p>The Amazon Resource Name (ARN) of the target table for which to retrieve integration table properties. Currently, this API only supports retrieving properties for target tables, and the provided ARN should be the ARN of the target table in the Glue Data Catalog. Support for retrieving integration table properties for source connections (using the connection ARN) is not yet implemented and will be added in a future release. </p>",
         "GetIntegrationTablePropertiesRequest$TableName": "<p>The name of the table to be replicated.</p>",
-        "GetIntegrationTablePropertiesResponse$ResourceArn": "<p>The connection ARN of the source, or the database ARN of the target.</p>",
+        "GetIntegrationTablePropertiesResponse$ResourceArn": "<p>The Amazon Resource Name (ARN) of the target table for which to retrieve integration table properties. Currently, this API only supports retrieving properties for target tables, and the provided ARN should be the ARN of the target table in the Glue Data Catalog. Support for retrieving integration table properties for source connections (using the connection ARN) is not yet implemented and will be added in a future release. </p>",
         "GetIntegrationTablePropertiesResponse$TableName": "<p>The name of the table to be replicated.</p>",
         "InboundIntegration$SourceArn": "<p>The ARN of the source resource for the integration.</p>",
         "InboundIntegration$TargetArn": "<p>The ARN of the target resource for the integration.</p>",
@@ -9548,6 +9548,7 @@
         "IntegrationFilterValues$member": null,
         "IntegrationPartition$FieldName": "<p>The field name used to partition data on the target. Avoid using columns that have unique values for each row (for example, `LastModifiedTimestamp`, `SystemModTimeStamp`) as the partition column. These columns are not suitable for partitioning because they create a large number of small partitions, which can lead to performance issues.</p>",
         "IntegrationPartition$FunctionSpec": "<p>Specifies the function used to partition data on the target. The only accepted value for this parameter is `'identity'` (string). The `'identity'` function ensures that the data partitioning on the target follows the same scheme as the source. In other words, the partitioning structure of the source data is preserved in the target destination.</p>",
+        "IntegrationPartition$ConversionSpec": "<p>Specifies the timestamp format of the source data. Valid values are:</p> <ul> <li> <p> <code>epoch_sec</code> - Unix epoch timestamp in seconds</p> </li> <li> <p> <code>epoch_milli</code> - Unix epoch timestamp in milliseconds</p> </li> <li> <p> <code>iso</code> - ISO 8601 formatted timestamp</p> </li> </ul> <note> <p> Only specify <code>ConversionSpec</code> when using timestamp-based partition functions (year, month, day, or hour). Glue Zero-ETL uses this parameter to correctly transform source data into timestamp format before partitioning. </p> <p> Do not use high-cardinality columns with the <code>identity</code> partition function. High-cardinality columns include: </p> <ul> <li> <p>Primary keys</p> </li> <li> <p>Timestamp fields (such as <code>LastModifiedTimestamp</code>, <code>CreatedDate</code>)</p> </li> <li> <p>System-generated timestamps</p> </li> </ul> <p> Using high-cardinality columns with identity partitioning creates many small partitions, which can significantly degrade ingestion performance. </p> </note>",
         "ModifyIntegrationRequest$IntegrationIdentifier": "<p>The Amazon Resource Name (ARN) for the integration.</p>",
         "ModifyIntegrationRequest$IntegrationName": "<p>A unique name for an integration in Glue.</p>",
         "ModifyIntegrationResponse$SourceArn": "<p>The ARN of the source for the integration.</p>",

generator/ServiceModels/glue/glue-2017-03-31.normal.json

@@ -9154,7 +9154,7 @@
       "members":{
         "ResourceArn":{
           "shape":"String128",
-          "documentation":"<p>The connection ARN of the source, or the database ARN of the target.</p>"
+          "documentation":"<p>The Amazon Resource Name (ARN) of the target table for which to create integration table properties. Currently, this API only supports creating integration table properties for target tables, and the provided ARN should be the ARN of the target table in the Glue Data Catalog. Support for creating integration table properties for source connections (using the connection ARN) is not yet implemented and will be added in a future release. </p>"
         },
         "TableName":{
           "shape":"String128",
@@ -14466,7 +14466,7 @@
       "members":{
         "ResourceArn":{
           "shape":"String128",
-          "documentation":"<p>The connection ARN of the source, or the database ARN of the target.</p>"
+          "documentation":"<p>The Amazon Resource Name (ARN) of the target table for which to retrieve integration table properties. Currently, this API only supports retrieving properties for target tables, and the provided ARN should be the ARN of the target table in the Glue Data Catalog. Support for retrieving integration table properties for source connections (using the connection ARN) is not yet implemented and will be added in a future release. </p>"
         },
         "TableName":{
           "shape":"String128",
@@ -14479,7 +14479,7 @@
       "members":{
         "ResourceArn":{
           "shape":"String128",
-          "documentation":"<p>The connection ARN of the source, or the database ARN of the target.</p>"
+          "documentation":"<p>The Amazon Resource Name (ARN) of the target table for which to retrieve integration table properties. Currently, this API only supports retrieving properties for target tables, and the provided ARN should be the ARN of the target table in the Glue Data Catalog. Support for retrieving integration table properties for source connections (using the connection ARN) is not yet implemented and will be added in a future release. </p>"
         },
         "TableName":{
           "shape":"String128",
@@ -16963,6 +16963,10 @@
         "FunctionSpec":{
           "shape":"String128",
           "documentation":"<p>Specifies the function used to partition data on the target. The only accepted value for this parameter is `'identity'` (string). The `'identity'` function ensures that the data partitioning on the target follows the same scheme as the source. In other words, the partitioning structure of the source data is preserved in the target destination.</p>"
+        },
+        "ConversionSpec":{
+          "shape":"String128",
+          "documentation":"<p>Specifies the timestamp format of the source data. Valid values are:</p> <ul> <li> <p> <code>epoch_sec</code> - Unix epoch timestamp in seconds</p> </li> <li> <p> <code>epoch_milli</code> - Unix epoch timestamp in milliseconds</p> </li> <li> <p> <code>iso</code> - ISO 8601 formatted timestamp</p> </li> </ul> <note> <p> Only specify <code>ConversionSpec</code> when using timestamp-based partition functions (year, month, day, or hour). Glue Zero-ETL uses this parameter to correctly transform source data into timestamp format before partitioning. </p> <p> Do not use high-cardinality columns with the <code>identity</code> partition function. High-cardinality columns include: </p> <ul> <li> <p>Primary keys</p> </li> <li> <p>Timestamp fields (such as <code>LastModifiedTimestamp</code>, <code>CreatedDate</code>)</p> </li> <li> <p>System-generated timestamps</p> </li> </ul> <p> Using high-cardinality columns with identity partitioning creates many small partitions, which can significantly degrade ingestion performance. </p> </note>"
         }
       },
       "documentation":"<p>A structure that describes how data is partitioned on the target.</p>"

sdk/code-analysis/ServiceAnalysis/Glue/Generated/PropertyValueRules.xml

@@ -5025,6 +5025,11 @@
     <min>1</min>
     <max>128</max>
   </property-value-rule>
+  <property-value-rule>
+    <property>Amazon.Glue.Model.IntegrationPartition.ConversionSpec</property>
+    <min>1</min>
+    <max>128</max>
+  </property-value-rule>
   <property-value-rule>
     <property>Amazon.Glue.Model.IntegrationPartition.FieldName</property>
     <min>1</min>

sdk/src/Services/Glue/Generated/Model/CreateIntegrationTablePropertiesRequest.cs

@@ -47,7 +47,12 @@ public partial class CreateIntegrationTablePropertiesRequest : AmazonGlueRequest
         /// <summary>
         /// Gets and sets the property ResourceArn. 
         /// <para>
-        /// The connection ARN of the source, or the database ARN of the target.
+        /// The Amazon Resource Name (ARN) of the target table for which to create integration
+        /// table properties. Currently, this API only supports creating integration table properties
+        /// for target tables, and the provided ARN should be the ARN of the target table in the
+        /// Glue Data Catalog. Support for creating integration table properties for source connections
+        /// (using the connection ARN) is not yet implemented and will be added in a future release.
+        /// 
         /// </para>
         /// </summary>
         [AWSProperty(Required=true, Min=1, Max=128)]

sdk/src/Services/Glue/Generated/Model/GetIntegrationTablePropertiesRequest.cs

@@ -43,7 +43,12 @@ public partial class GetIntegrationTablePropertiesRequest : AmazonGlueRequest
         /// <summary>
         /// Gets and sets the property ResourceArn. 
         /// <para>
-        /// The connection ARN of the source, or the database ARN of the target.
+        /// The Amazon Resource Name (ARN) of the target table for which to retrieve integration
+        /// table properties. Currently, this API only supports retrieving properties for target
+        /// tables, and the provided ARN should be the ARN of the target table in the Glue Data
+        /// Catalog. Support for retrieving integration table properties for source connections
+        /// (using the connection ARN) is not yet implemented and will be added in a future release.
+        /// 
         /// </para>
         /// </summary>
         [AWSProperty(Required=true, Min=1, Max=128)]

sdk/src/Services/Glue/Generated/Model/GetIntegrationTablePropertiesResponse.cs

@@ -42,7 +42,12 @@ public partial class GetIntegrationTablePropertiesResponse : AmazonWebServiceRes
         /// <summary>
         /// Gets and sets the property ResourceArn. 
         /// <para>
-        /// The connection ARN of the source, or the database ARN of the target.
+        /// The Amazon Resource Name (ARN) of the target table for which to retrieve integration
+        /// table properties. Currently, this API only supports retrieving properties for target
+        /// tables, and the provided ARN should be the ARN of the target table in the Glue Data
+        /// Catalog. Support for retrieving integration table properties for source connections
+        /// (using the connection ARN) is not yet implemented and will be added in a future release.
+        /// 
         /// </para>
         /// </summary>
         [AWSProperty(Min=1, Max=128)]

sdk/src/Services/Glue/Generated/Model/IntegrationPartition.cs

@@ -34,9 +34,70 @@ namespace Amazon.Glue.Model
     /// </summary>
     public partial class IntegrationPartition
     {
+        private string _conversionSpec;
         private string _fieldName;
         private string _functionSpec;
 
+        /// <summary>
+        /// Gets and sets the property ConversionSpec. 
+        /// <para>
+        /// Specifies the timestamp format of the source data. Valid values are:
+        /// </para>
+        ///  <ul> <li> 
+        /// <para>
+        ///  <c>epoch_sec</c> - Unix epoch timestamp in seconds
+        /// </para>
+        ///  </li> <li> 
+        /// <para>
+        ///  <c>epoch_milli</c> - Unix epoch timestamp in milliseconds
+        /// </para>
+        ///  </li> <li> 
+        /// <para>
+        ///  <c>iso</c> - ISO 8601 formatted timestamp
+        /// </para>
+        ///  </li> </ul> <note> 
+        /// <para>
+        ///  Only specify <c>ConversionSpec</c> when using timestamp-based partition functions
+        /// (year, month, day, or hour). Glue Zero-ETL uses this parameter to correctly transform
+        /// source data into timestamp format before partitioning. 
+        /// </para>
+        ///  
+        /// <para>
+        ///  Do not use high-cardinality columns with the <c>identity</c> partition function.
+        /// High-cardinality columns include: 
+        /// </para>
+        ///  <ul> <li> 
+        /// <para>
+        /// Primary keys
+        /// </para>
+        ///  </li> <li> 
+        /// <para>
+        /// Timestamp fields (such as <c>LastModifiedTimestamp</c>, <c>CreatedDate</c>)
+        /// </para>
+        ///  </li> <li> 
+        /// <para>
+        /// System-generated timestamps
+        /// </para>
+        ///  </li> </ul> 
+        /// <para>
+        ///  Using high-cardinality columns with identity partitioning creates many small partitions,
+        /// which can significantly degrade ingestion performance. 
+        /// </para>
+        ///  </note>
+        /// </summary>
+        [AWSProperty(Min=1, Max=128)]
+        public string ConversionSpec
+        {
+            get { return this._conversionSpec; }
+            set { this._conversionSpec = value; }
+        }
+
+        // Check to see if ConversionSpec property is set
+        internal bool IsSetConversionSpec()
+        {
+            return this._conversionSpec != null;
+        }
+
         /// <summary>
         /// Gets and sets the property FieldName. 
         /// <para>

sdk/src/Services/Glue/Generated/Model/Internal/MarshallTransformations/IntegrationPartitionMarshaller.cs

@@ -48,6 +48,12 @@ public void Marshall(IntegrationPartition requestObject, JsonMarshallerContext c
         {
             if(requestObject == null)
                 return;
+            if(requestObject.IsSetConversionSpec())
+            {
+                context.Writer.WritePropertyName("ConversionSpec");
+                context.Writer.Write(requestObject.ConversionSpec);
+            }
+
             if(requestObject.IsSetFieldName())
             {
                 context.Writer.WritePropertyName("FieldName");

sdk/src/Services/Glue/Generated/Model/Internal/MarshallTransformations/IntegrationPartitionUnmarshaller.cs

@@ -66,6 +66,12 @@ public IntegrationPartition Unmarshall(JsonUnmarshallerContext context)
             int targetDepth = context.CurrentDepth;
             while (context.ReadAtDepth(targetDepth))
             {
+                if (context.TestExpression("ConversionSpec", targetDepth))
+                {
+                    var unmarshaller = StringUnmarshaller.Instance;
+                    unmarshalledObject.ConversionSpec = unmarshaller.Unmarshall(context);
+                    continue;
+                }
                 if (context.TestExpression("FieldName", targetDepth))
                 {
                     var unmarshaller = StringUnmarshaller.Instance;