Skip to content

Commit eeecd10

Browse files
author
awstools
committed
feat(client-glue): This release supports additional ConversionSpec parameter as part of IntegrationPartition Structure in CreateIntegrationTableProperty API. This parameter is referred to apply appropriate column transformation for columns that are used for timestamp based partitioning
1 parent f5e3e09 commit eeecd10

File tree

6 files changed

+67
-6
lines changed

6 files changed

+67
-6
lines changed

‎clients/client-glue/src/commands/CreateIntegrationTablePropertiesCommand.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ export interface CreateIntegrationTablePropertiesCommandOutput
5959
* { // IntegrationPartition
6060
* FieldName: "STRING_VALUE",
6161
* FunctionSpec: "STRING_VALUE",
62+
* ConversionSpec: "STRING_VALUE",
6263
* },
6364
* ],
6465
* TargetTableName: "STRING_VALUE",

‎clients/client-glue/src/commands/GetIntegrationTablePropertiesCommand.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ export interface GetIntegrationTablePropertiesCommandOutput
6565
* // { // IntegrationPartition
6666
* // FieldName: "STRING_VALUE",
6767
* // FunctionSpec: "STRING_VALUE",
68+
* // ConversionSpec: "STRING_VALUE",
6869
* // },
6970
* // ],
7071
* // TargetTableName: "STRING_VALUE",

‎clients/client-glue/src/commands/UpdateIntegrationTablePropertiesCommand.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ export interface UpdateIntegrationTablePropertiesCommandOutput
6060
* { // IntegrationPartition
6161
* FieldName: "STRING_VALUE",
6262
* FunctionSpec: "STRING_VALUE",
63+
* ConversionSpec: "STRING_VALUE",
6364
* },
6465
* ],
6566
* TargetTableName: "STRING_VALUE",

‎clients/client-glue/src/models/models_1.ts‎

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2729,6 +2729,49 @@ export interface IntegrationPartition {
27292729
* @public
27302730
*/
27312731
FunctionSpec?: string | undefined;
2732+
2733+
/**
2734+
* <p>Specifies the timestamp format of the source data. Valid values are:</p>
2735+
* <ul>
2736+
* <li>
2737+
* <p>
2738+
* <code>epoch_sec</code> - Unix epoch timestamp in seconds</p>
2739+
* </li>
2740+
* <li>
2741+
* <p>
2742+
* <code>epoch_milli</code> - Unix epoch timestamp in milliseconds</p>
2743+
* </li>
2744+
* <li>
2745+
* <p>
2746+
* <code>iso</code> - ISO 8601 formatted timestamp</p>
2747+
* </li>
2748+
* </ul>
2749+
* <note>
2750+
* <p>
2751+
* Only specify <code>ConversionSpec</code> when using timestamp-based partition functions (year, month, day, or hour).
2752+
* Glue Zero-ETL uses this parameter to correctly transform source data into timestamp format before partitioning.
2753+
* </p>
2754+
* <p>
2755+
* Do not use high-cardinality columns with the <code>identity</code> partition function. High-cardinality columns include:
2756+
* </p>
2757+
* <ul>
2758+
* <li>
2759+
* <p>Primary keys</p>
2760+
* </li>
2761+
* <li>
2762+
* <p>Timestamp fields (such as <code>LastModifiedTimestamp</code>, <code>CreatedDate</code>)</p>
2763+
* </li>
2764+
* <li>
2765+
* <p>System-generated timestamps</p>
2766+
* </li>
2767+
* </ul>
2768+
* <p>
2769+
* Using high-cardinality columns with identity partitioning creates many small partitions, which can significantly degrade ingestion performance.
2770+
* </p>
2771+
* </note>
2772+
* @public
2773+
*/
2774+
ConversionSpec?: string | undefined;
27322775
}
27332776

27342777
/**
@@ -2775,7 +2818,10 @@ export interface TargetTableConfig {
27752818
*/
27762819
export interface CreateIntegrationTablePropertiesRequest {
27772820
/**
2778-
* <p>The connection ARN of the source, or the database ARN of the target.</p>
2821+
* <p>The Amazon Resource Name (ARN) of the target table for which to create integration table properties. Currently, this API only supports creating
2822+
* integration table properties for target tables, and the provided ARN should be the ARN of the target table in the Glue Data Catalog. Support for
2823+
* creating integration table properties for source connections (using the connection ARN) is not yet implemented and will be added in a future release.
2824+
* </p>
27792825
* @public
27802826
*/
27812827
ResourceArn: string | undefined;

‎clients/client-glue/src/models/models_2.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2111,7 +2111,10 @@ export interface GetIntegrationResourcePropertyResponse {
21112111
*/
21122112
export interface GetIntegrationTablePropertiesRequest {
21132113
/**
2114-
* <p>The connection ARN of the source, or the database ARN of the target.</p>
2114+
* <p>The Amazon Resource Name (ARN) of the target table for which to retrieve integration table properties. Currently, this API only supports retrieving
2115+
* properties for target tables, and the provided ARN should be the ARN of the target table in the Glue Data Catalog. Support for retrieving integration
2116+
* table properties for source connections (using the connection ARN) is not yet implemented and will be added in a future release.
2117+
* </p>
21152118
* @public
21162119
*/
21172120
ResourceArn: string | undefined;
@@ -2128,7 +2131,10 @@ export interface GetIntegrationTablePropertiesRequest {
21282131
*/
21292132
export interface GetIntegrationTablePropertiesResponse {
21302133
/**
2131-
* <p>The connection ARN of the source, or the database ARN of the target.</p>
2134+
* <p>The Amazon Resource Name (ARN) of the target table for which to retrieve integration table properties. Currently, this API only supports retrieving
2135+
* properties for target tables, and the provided ARN should be the ARN of the target table in the Glue Data Catalog. Support for retrieving integration
2136+
* table properties for source connections (using the connection ARN) is not yet implemented and will be added in a future release.
2137+
* </p>
21322138
* @public
21332139
*/
21342140
ResourceArn?: string | undefined;

‎codegen/sdk-codegen/aws-models/glue.json‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10699,7 +10699,7 @@
1069910699
"ResourceArn": {
1070010700
"target": "com.amazonaws.glue#String128",
1070110701
"traits": {
10702-
"smithy.api#documentation": "<p>The connection ARN of the source, or the database ARN of the target.</p>",
10702+
"smithy.api#documentation": "<p>The Amazon Resource Name (ARN) of the target table for which to create integration table properties. Currently, this API only supports creating \n integration table properties for target tables, and the provided ARN should be the ARN of the target table in the Glue Data Catalog. Support for \n creating integration table properties for source connections (using the connection ARN) is not yet implemented and will be added in a future release.\n </p>",
1070310703
"smithy.api#required": {}
1070410704
}
1070510705
},
@@ -21400,7 +21400,7 @@
2140021400
"ResourceArn": {
2140121401
"target": "com.amazonaws.glue#String128",
2140221402
"traits": {
21403-
"smithy.api#documentation": "<p>The connection ARN of the source, or the database ARN of the target.</p>",
21403+
"smithy.api#documentation": "<p>The Amazon Resource Name (ARN) of the target table for which to retrieve integration table properties. Currently, this API only supports retrieving \n properties for target tables, and the provided ARN should be the ARN of the target table in the Glue Data Catalog. Support for retrieving integration \n table properties for source connections (using the connection ARN) is not yet implemented and will be added in a future release.\n </p>",
2140421404
"smithy.api#required": {}
2140521405
}
2140621406
},
@@ -21422,7 +21422,7 @@
2142221422
"ResourceArn": {
2142321423
"target": "com.amazonaws.glue#String128",
2142421424
"traits": {
21425-
"smithy.api#documentation": "<p>The connection ARN of the source, or the database ARN of the target.</p>"
21425+
"smithy.api#documentation": "<p>The Amazon Resource Name (ARN) of the target table for which to retrieve integration table properties. Currently, this API only supports retrieving \n properties for target tables, and the provided ARN should be the ARN of the target table in the Glue Data Catalog. Support for retrieving integration \n table properties for source connections (using the connection ARN) is not yet implemented and will be added in a future release.\n </p>"
2142621426
}
2142721427
},
2142821428
"TableName": {
@@ -26506,6 +26506,12 @@
2650626506
"traits": {
2650726507
"smithy.api#documentation": "<p>Specifies the function used to partition data on the target. The only accepted value for this parameter is `'identity'` (string). \n The `'identity'` function ensures that the data partitioning on the target follows the same scheme as the source. In other words, the partitioning \n structure of the source data is preserved in the target destination.</p>"
2650826508
}
26509+
},
26510+
"ConversionSpec": {
26511+
"target": "com.amazonaws.glue#String128",
26512+
"traits": {
26513+
"smithy.api#documentation": "<p>Specifies the timestamp format of the source data. Valid values are:</p>\n <ul>\n <li>\n <p>\n <code>epoch_sec</code> - Unix epoch timestamp in seconds</p>\n </li>\n <li>\n <p>\n <code>epoch_milli</code> - Unix epoch timestamp in milliseconds</p>\n </li>\n <li>\n <p>\n <code>iso</code> - ISO 8601 formatted timestamp</p>\n </li>\n </ul>\n <note>\n <p>\n Only specify <code>ConversionSpec</code> when using timestamp-based partition functions (year, month, day, or hour). \n Glue Zero-ETL uses this parameter to correctly transform source data into timestamp format before partitioning.\n </p>\n <p>\n Do not use high-cardinality columns with the <code>identity</code> partition function. High-cardinality columns include:\n </p>\n <ul>\n <li>\n <p>Primary keys</p>\n </li>\n <li>\n <p>Timestamp fields (such as <code>LastModifiedTimestamp</code>, <code>CreatedDate</code>)</p>\n </li>\n <li>\n <p>System-generated timestamps</p>\n </li>\n </ul>\n <p>\n Using high-cardinality columns with identity partitioning creates many small partitions, which can significantly degrade ingestion performance.\n </p>\n </note>"
26514+
}
2650926515
}
2651026516
},
2651126517
"traits": {

0 commit comments

Comments
 (0)