Skip to content

Commit 5451046

Browse files
author
awstools
committed
feat(client-glue): AWS Glue Data Catalog now supports Iceberg Optimization settings at the Catalog level, and supports new options to control the optimization job run rate.
1 parent 4f02729 commit 5451046

16 files changed

+625
-280
lines changed

‎clients/client-glue/src/commands/BatchGetTableOptimizerCommand.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,19 +64,23 @@ export interface BatchGetTableOptimizerCommandOutput extends BatchGetTableOptimi
6464
* // compactionConfiguration: { // CompactionConfiguration
6565
* // icebergConfiguration: { // IcebergCompactionConfiguration
6666
* // strategy: "binpack" || "sort" || "z-order",
67+
* // minInputFiles: Number("int"),
68+
* // deleteFileThreshold: Number("int"),
6769
* // },
6870
* // },
6971
* // retentionConfiguration: { // RetentionConfiguration
7072
* // icebergConfiguration: { // IcebergRetentionConfiguration
7173
* // snapshotRetentionPeriodInDays: Number("int"),
7274
* // numberOfSnapshotsToRetain: Number("int"),
7375
* // cleanExpiredFiles: true || false,
76+
* // runRateInHours: Number("int"),
7477
* // },
7578
* // },
7679
* // orphanFileDeletionConfiguration: { // OrphanFileDeletionConfiguration
7780
* // icebergConfiguration: { // IcebergOrphanFileDeletionConfiguration
7881
* // orphanFileRetentionPeriodInDays: Number("int"),
7982
* // location: "STRING_VALUE",
83+
* // runRateInHours: Number("int"),
8084
* // },
8185
* // },
8286
* // },
@@ -120,6 +124,7 @@ export interface BatchGetTableOptimizerCommandOutput extends BatchGetTableOptimi
120124
* // },
121125
* // },
122126
* // },
127+
* // configurationSource: "catalog" || "table",
123128
* // },
124129
* // },
125130
* // ],

‎clients/client-glue/src/commands/CreateCatalogCommand.ts‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,19 @@ export interface CreateCatalogCommandOutput extends CreateCatalogResponse, __Met
5757
* KmsKey: "STRING_VALUE",
5858
* CatalogType: "STRING_VALUE",
5959
* },
60-
* CustomProperties: {
61-
* "<keys>": "STRING_VALUE",
60+
* IcebergOptimizationProperties: { // IcebergOptimizationProperties
61+
* RoleArn: "STRING_VALUE",
62+
* Compaction: {
63+
* "<keys>": "STRING_VALUE",
64+
* },
65+
* Retention: {
66+
* "<keys>": "STRING_VALUE",
67+
* },
68+
* OrphanFileDeletion: {
69+
* "<keys>": "STRING_VALUE",
70+
* },
6271
* },
72+
* CustomProperties: "<ParametersMap>",
6373
* },
6474
* CreateTableDefaultPermissions: [ // PrincipalPermissionsList
6575
* { // PrincipalPermissions

‎clients/client-glue/src/commands/CreateTableOptimizerCommand.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,19 +49,23 @@ export interface CreateTableOptimizerCommandOutput extends CreateTableOptimizerR
4949
* compactionConfiguration: { // CompactionConfiguration
5050
* icebergConfiguration: { // IcebergCompactionConfiguration
5151
* strategy: "binpack" || "sort" || "z-order",
52+
* minInputFiles: Number("int"),
53+
* deleteFileThreshold: Number("int"),
5254
* },
5355
* },
5456
* retentionConfiguration: { // RetentionConfiguration
5557
* icebergConfiguration: { // IcebergRetentionConfiguration
5658
* snapshotRetentionPeriodInDays: Number("int"),
5759
* numberOfSnapshotsToRetain: Number("int"),
5860
* cleanExpiredFiles: true || false,
61+
* runRateInHours: Number("int"),
5962
* },
6063
* },
6164
* orphanFileDeletionConfiguration: { // OrphanFileDeletionConfiguration
6265
* icebergConfiguration: { // IcebergOrphanFileDeletionConfiguration
6366
* orphanFileRetentionPeriodInDays: Number("int"),
6467
* location: "STRING_VALUE",
68+
* runRateInHours: Number("int"),
6569
* },
6670
* },
6771
* },

‎clients/client-glue/src/commands/GetCatalogCommand.ts‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,20 @@ export interface GetCatalogCommandOutput extends GetCatalogResponse, __MetadataB
7070
* // StatusMessage: "STRING_VALUE",
7171
* // CatalogType: "STRING_VALUE",
7272
* // },
73-
* // CustomProperties: {
74-
* // "<keys>": "STRING_VALUE",
73+
* // IcebergOptimizationProperties: { // IcebergOptimizationPropertiesOutput
74+
* // RoleArn: "STRING_VALUE",
75+
* // Compaction: {
76+
* // "<keys>": "STRING_VALUE",
77+
* // },
78+
* // Retention: {
79+
* // "<keys>": "STRING_VALUE",
80+
* // },
81+
* // OrphanFileDeletion: {
82+
* // "<keys>": "STRING_VALUE",
83+
* // },
84+
* // LastUpdatedTime: new Date("TIMESTAMP"),
7585
* // },
86+
* // CustomProperties: "<ParametersMap>",
7687
* // },
7788
* // CreateTableDefaultPermissions: [ // PrincipalPermissionsList
7889
* // { // PrincipalPermissions

‎clients/client-glue/src/commands/GetCatalogsCommand.ts‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,20 @@ export interface GetCatalogsCommandOutput extends GetCatalogsResponse, __Metadat
7575
* // StatusMessage: "STRING_VALUE",
7676
* // CatalogType: "STRING_VALUE",
7777
* // },
78-
* // CustomProperties: {
79-
* // "<keys>": "STRING_VALUE",
78+
* // IcebergOptimizationProperties: { // IcebergOptimizationPropertiesOutput
79+
* // RoleArn: "STRING_VALUE",
80+
* // Compaction: {
81+
* // "<keys>": "STRING_VALUE",
82+
* // },
83+
* // Retention: {
84+
* // "<keys>": "STRING_VALUE",
85+
* // },
86+
* // OrphanFileDeletion: {
87+
* // "<keys>": "STRING_VALUE",
88+
* // },
89+
* // LastUpdatedTime: new Date("TIMESTAMP"),
8090
* // },
91+
* // CustomProperties: "<ParametersMap>",
8192
* // },
8293
* // CreateTableDefaultPermissions: [ // PrincipalPermissionsList
8394
* // { // PrincipalPermissions

‎clients/client-glue/src/commands/GetTableOptimizerCommand.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,19 +58,23 @@ export interface GetTableOptimizerCommandOutput extends GetTableOptimizerRespons
5858
* // compactionConfiguration: { // CompactionConfiguration
5959
* // icebergConfiguration: { // IcebergCompactionConfiguration
6060
* // strategy: "binpack" || "sort" || "z-order",
61+
* // minInputFiles: Number("int"),
62+
* // deleteFileThreshold: Number("int"),
6163
* // },
6264
* // },
6365
* // retentionConfiguration: { // RetentionConfiguration
6466
* // icebergConfiguration: { // IcebergRetentionConfiguration
6567
* // snapshotRetentionPeriodInDays: Number("int"),
6668
* // numberOfSnapshotsToRetain: Number("int"),
6769
* // cleanExpiredFiles: true || false,
70+
* // runRateInHours: Number("int"),
6871
* // },
6972
* // },
7073
* // orphanFileDeletionConfiguration: { // OrphanFileDeletionConfiguration
7174
* // icebergConfiguration: { // IcebergOrphanFileDeletionConfiguration
7275
* // orphanFileRetentionPeriodInDays: Number("int"),
7376
* // location: "STRING_VALUE",
77+
* // runRateInHours: Number("int"),
7478
* // },
7579
* // },
7680
* // },
@@ -114,6 +118,7 @@ export interface GetTableOptimizerCommandOutput extends GetTableOptimizerRespons
114118
* // },
115119
* // },
116120
* // },
121+
* // configurationSource: "catalog" || "table",
117122
* // },
118123
* // };
119124
*

‎clients/client-glue/src/commands/ListWorkflowsCommand.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { MetadataBearer as __MetadataBearer } from "@smithy/types";
66

77
import { commonParams } from "../endpoint/EndpointParameters";
88
import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
9-
import { ListWorkflowsRequest, ListWorkflowsResponse } from "../models/models_2";
9+
import { ListWorkflowsRequest, ListWorkflowsResponse } from "../models/models_3";
1010
import { de_ListWorkflowsCommand, se_ListWorkflowsCommand } from "../protocols/Aws_json1_1";
1111

1212
/**

‎clients/client-glue/src/commands/ModifyIntegrationCommand.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,7 @@ import { MetadataBearer as __MetadataBearer } from "@smithy/types";
66

77
import { commonParams } from "../endpoint/EndpointParameters";
88
import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
9-
import { ModifyIntegrationRequest } from "../models/models_2";
10-
import { ModifyIntegrationResponse } from "../models/models_3";
9+
import { ModifyIntegrationRequest, ModifyIntegrationResponse } from "../models/models_3";
1110
import { de_ModifyIntegrationCommand, se_ModifyIntegrationCommand } from "../protocols/Aws_json1_1";
1211

1312
/**

‎clients/client-glue/src/commands/UpdateCatalogCommand.ts‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,19 @@ export interface UpdateCatalogCommandOutput extends UpdateCatalogResponse, __Met
5757
* KmsKey: "STRING_VALUE",
5858
* CatalogType: "STRING_VALUE",
5959
* },
60-
* CustomProperties: {
61-
* "<keys>": "STRING_VALUE",
60+
* IcebergOptimizationProperties: { // IcebergOptimizationProperties
61+
* RoleArn: "STRING_VALUE",
62+
* Compaction: {
63+
* "<keys>": "STRING_VALUE",
64+
* },
65+
* Retention: {
66+
* "<keys>": "STRING_VALUE",
67+
* },
68+
* OrphanFileDeletion: {
69+
* "<keys>": "STRING_VALUE",
70+
* },
6271
* },
72+
* CustomProperties: "<ParametersMap>",
6373
* },
6474
* CreateTableDefaultPermissions: [ // PrincipalPermissionsList
6575
* { // PrincipalPermissions

‎clients/client-glue/src/commands/UpdateTableOptimizerCommand.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,19 +49,23 @@ export interface UpdateTableOptimizerCommandOutput extends UpdateTableOptimizerR
4949
* compactionConfiguration: { // CompactionConfiguration
5050
* icebergConfiguration: { // IcebergCompactionConfiguration
5151
* strategy: "binpack" || "sort" || "z-order",
52+
* minInputFiles: Number("int"),
53+
* deleteFileThreshold: Number("int"),
5254
* },
5355
* },
5456
* retentionConfiguration: { // RetentionConfiguration
5557
* icebergConfiguration: { // IcebergRetentionConfiguration
5658
* snapshotRetentionPeriodInDays: Number("int"),
5759
* numberOfSnapshotsToRetain: Number("int"),
5860
* cleanExpiredFiles: true || false,
61+
* runRateInHours: Number("int"),
5962
* },
6063
* },
6164
* orphanFileDeletionConfiguration: { // OrphanFileDeletionConfiguration
6265
* icebergConfiguration: { // IcebergOrphanFileDeletionConfiguration
6366
* orphanFileRetentionPeriodInDays: Number("int"),
6467
* location: "STRING_VALUE",
68+
* runRateInHours: Number("int"),
6569
* },
6670
* },
6771
* },

0 commit comments

Comments
 (0)