AWS Glue now supports sort and z-order strategy for managed automated compaction for Iceberg tables in addition to binpack.

Author: aws-sdk-dotnet-automation

generator/ServiceModels/glue/glue-2017-03-31.api.json

@@ -5720,12 +5720,26 @@
       "min":1,
       "pattern":"[\\u0020-\\uD7FF\\uE000-\\uFFFD\\uD800\\uDC00-\\uDBFF\\uDFFF\\t]*"
     },
+    "CompactionConfiguration":{
+      "type":"structure",
+      "members":{
+        "icebergConfiguration":{"shape":"IcebergCompactionConfiguration"}
+      }
+    },
     "CompactionMetrics":{
       "type":"structure",
       "members":{
         "IcebergMetrics":{"shape":"IcebergCompactionMetrics"}
       }
     },
+    "CompactionStrategy":{
+      "type":"string",
+      "enum":[
+        "binpack",
+        "sort",
+        "z-order"
+      ]
+    },
     "Comparator":{
       "type":"string",
       "enum":[
@@ -10623,6 +10637,12 @@
       "type":"string",
       "pattern":"^arn:aws(-(cn|us-gov|iso(-[bef])?))?:iam::[0-9]{12}:role/.+"
     },
+    "IcebergCompactionConfiguration":{
+      "type":"structure",
+      "members":{
+        "strategy":{"shape":"CompactionStrategy"}
+      }
+    },
     "IcebergCompactionMetrics":{
       "type":"structure",
       "members":{
@@ -15192,6 +15212,7 @@
         "roleArn":{"shape":"ArnString"},
         "enabled":{"shape":"NullableBoolean"},
         "vpcConfiguration":{"shape":"TableOptimizerVpcConfiguration"},
+        "compactionConfiguration":{"shape":"CompactionConfiguration"},
         "retentionConfiguration":{"shape":"RetentionConfiguration"},
         "orphanFileDeletionConfiguration":{"shape":"OrphanFileDeletionConfiguration"}
       }
@@ -15218,6 +15239,7 @@
         },
         "error":{"shape":"MessageString"},
         "compactionMetrics":{"shape":"CompactionMetrics"},
+        "compactionStrategy":{"shape":"CompactionStrategy"},
         "retentionMetrics":{"shape":"RetentionMetrics"},
         "orphanFileDeletionMetrics":{"shape":"OrphanFileDeletionMetrics"}
       }

generator/ServiceModels/glue/glue-2017-03-31.docs.json

@@ -1686,12 +1686,25 @@
         "UpdateSourceControlFromJobRequest$CommitId": "<p>A commit ID for a commit in the remote repository.</p>"
       }
     },
+    "CompactionConfiguration": {
+      "base": "<p>The configuration for a compaction optimizer. This configuration defines how data files in your table will be compacted to improve query performance and reduce storage costs.</p>",
+      "refs": {
+        "TableOptimizerConfiguration$compactionConfiguration": "<p>The configuration for a compaction optimizer. This configuration defines how data files in your table will be compacted to improve query performance and reduce storage costs.</p>"
+      }
+    },
     "CompactionMetrics": {
       "base": "<p>A structure that contains compaction metrics for the optimizer run.</p>",
       "refs": {
         "TableOptimizerRun$compactionMetrics": "<p>A <code>CompactionMetrics</code> object containing metrics for the optimizer run.</p>"
       }
     },
+    "CompactionStrategy": {
+      "base": null,
+      "refs": {
+        "IcebergCompactionConfiguration$strategy": "<p>The strategy to use for compaction. Valid values are:</p> <ul> <li> <p> <code>binpack</code>: Combines small files into larger files, typically targeting sizes over 100MB, while applying any pending deletes. This is the recommended compaction strategy for most use cases. </p> </li> <li> <p> <code>sort</code>: Organizes data based on specified columns which are sorted hierarchically during compaction, improving query performance for filtered operations. This strategy is recommended when your queries frequently filter on specific columns. To use this strategy, you must first define a sort order in your Iceberg table properties using the <code>sort_order</code> table property.</p> </li> <li> <p> <code>z-order</code>: Optimizes data organization by blending multiple attributes into a single scalar value that can be used for sorting, allowing efficient querying across multiple dimensions. This strategy is recommended when you need to query data across multiple dimensions simultaneously. To use this strategy, you must first define a sort order in your Iceberg table properties using the <code>sort_order</code> table property. </p> </li> </ul> <p>If an input is not provided, the default value 'binpack' will be used.</p>",
+        "TableOptimizerRun$compactionStrategy": "<p>The strategy used for the compaction run. Indicates which algorithm was applied to determine how files were selected and combined during the compaction process. Valid values are:</p> <ul> <li> <p> <code>binpack</code>: Combines small files into larger files, typically targeting sizes over 100MB, while applying any pending deletes. This is the recommended compaction strategy for most use cases. </p> </li> <li> <p> <code>sort</code>: Organizes data based on specified columns which are sorted hierarchically during compaction, improving query performance for filtered operations. This strategy is recommended when your queries frequently filter on specific columns. To use this strategy, you must first define a sort order in your Iceberg table properties using the <code>sort_order</code> table property.</p> </li> <li> <p> <code>z-order</code>: Optimizes data organization by blending multiple attributes into a single scalar value that can be used for sorting, allowing efficient querying across multiple dimensions. This strategy is recommended when you need to query data across multiple dimensions simultaneously. To use this strategy, you must first define a sort order in your Iceberg table properties using the <code>sort_order</code> table property. </p> </li> </ul>"
+      }
+    },
     "Comparator": {
       "base": null,
       "refs": {
@@ -5267,6 +5280,12 @@
         "EncryptionAtRest$CatalogEncryptionServiceRole": "<p>The role that Glue assumes to encrypt and decrypt the Data Catalog objects on the caller's behalf.</p>"
       }
     },
+    "IcebergCompactionConfiguration": {
+      "base": "<p>The configuration for an Iceberg compaction optimizer. This configuration defines parameters for optimizing the layout of data files in Iceberg tables.</p>",
+      "refs": {
+        "CompactionConfiguration$icebergConfiguration": "<p>The configuration for an Iceberg compaction optimizer.</p>"
+      }
+    },
     "IcebergCompactionMetrics": {
       "base": "<p>Compaction metrics for Iceberg for the optimizer run.</p>",
       "refs": {

generator/ServiceModels/glue/glue-2017-03-31.normal.json

@@ -7349,6 +7349,16 @@
       "min":1,
       "pattern":"[\\u0020-\\uD7FF\\uE000-\\uFFFD\\uD800\\uDC00-\\uDBFF\\uDFFF\\t]*"
     },
+    "CompactionConfiguration":{
+      "type":"structure",
+      "members":{
+        "icebergConfiguration":{
+          "shape":"IcebergCompactionConfiguration",
+          "documentation":"<p>The configuration for an Iceberg compaction optimizer.</p>"
+        }
+      },
+      "documentation":"<p>The configuration for a compaction optimizer. This configuration defines how data files in your table will be compacted to improve query performance and reduce storage costs.</p>"
+    },
     "CompactionMetrics":{
       "type":"structure",
       "members":{
@@ -7359,6 +7369,14 @@
       },
       "documentation":"<p>A structure that contains compaction metrics for the optimizer run.</p>"
     },
+    "CompactionStrategy":{
+      "type":"string",
+      "enum":[
+        "binpack",
+        "sort",
+        "z-order"
+      ]
+    },
     "Comparator":{
       "type":"string",
       "enum":[
@@ -16545,6 +16563,16 @@
       "type":"string",
       "pattern":"^arn:aws(-(cn|us-gov|iso(-[bef])?))?:iam::[0-9]{12}:role/.+"
     },
+    "IcebergCompactionConfiguration":{
+      "type":"structure",
+      "members":{
+        "strategy":{
+          "shape":"CompactionStrategy",
+          "documentation":"<p>The strategy to use for compaction. Valid values are:</p> <ul> <li> <p> <code>binpack</code>: Combines small files into larger files, typically targeting sizes over 100MB, while applying any pending deletes. This is the recommended compaction strategy for most use cases. </p> </li> <li> <p> <code>sort</code>: Organizes data based on specified columns which are sorted hierarchically during compaction, improving query performance for filtered operations. This strategy is recommended when your queries frequently filter on specific columns. To use this strategy, you must first define a sort order in your Iceberg table properties using the <code>sort_order</code> table property.</p> </li> <li> <p> <code>z-order</code>: Optimizes data organization by blending multiple attributes into a single scalar value that can be used for sorting, allowing efficient querying across multiple dimensions. This strategy is recommended when you need to query data across multiple dimensions simultaneously. To use this strategy, you must first define a sort order in your Iceberg table properties using the <code>sort_order</code> table property. </p> </li> </ul> <p>If an input is not provided, the default value 'binpack' will be used.</p>"
+        }
+      },
+      "documentation":"<p>The configuration for an Iceberg compaction optimizer. This configuration defines parameters for optimizing the layout of data files in Iceberg tables.</p>"
+    },
     "IcebergCompactionMetrics":{
       "type":"structure",
       "members":{
@@ -24936,6 +24964,10 @@
           "shape":"TableOptimizerVpcConfiguration",
           "documentation":"<p>A <code>TableOptimizerVpcConfiguration</code> object representing the VPC configuration for a table optimizer.</p> <p>This configuration is necessary to perform optimization on tables that are in a customer VPC.</p>"
         },
+        "compactionConfiguration":{
+          "shape":"CompactionConfiguration",
+          "documentation":"<p>The configuration for a compaction optimizer. This configuration defines how data files in your table will be compacted to improve query performance and reduce storage costs.</p>"
+        },
         "retentionConfiguration":{
           "shape":"RetentionConfiguration",
           "documentation":"<p>The configuration for a snapshot retention optimizer.</p>"
@@ -24985,6 +25017,10 @@
           "shape":"CompactionMetrics",
           "documentation":"<p>A <code>CompactionMetrics</code> object containing metrics for the optimizer run.</p>"
         },
+        "compactionStrategy":{
+          "shape":"CompactionStrategy",
+          "documentation":"<p>The strategy used for the compaction run. Indicates which algorithm was applied to determine how files were selected and combined during the compaction process. Valid values are:</p> <ul> <li> <p> <code>binpack</code>: Combines small files into larger files, typically targeting sizes over 100MB, while applying any pending deletes. This is the recommended compaction strategy for most use cases. </p> </li> <li> <p> <code>sort</code>: Organizes data based on specified columns which are sorted hierarchically during compaction, improving query performance for filtered operations. This strategy is recommended when your queries frequently filter on specific columns. To use this strategy, you must first define a sort order in your Iceberg table properties using the <code>sort_order</code> table property.</p> </li> <li> <p> <code>z-order</code>: Optimizes data organization by blending multiple attributes into a single scalar value that can be used for sorting, allowing efficient querying across multiple dimensions. This strategy is recommended when you need to query data across multiple dimensions simultaneously. To use this strategy, you must first define a sort order in your Iceberg table properties using the <code>sort_order</code> table property. </p> </li> </ul>"
+        },
         "retentionMetrics":{
           "shape":"RetentionMetrics",
           "documentation":"<p>A <code>RetentionMetrics</code> object containing metrics for the optimizer run.</p>"

sdk/src/Services/Glue/Generated/Model/CompactionConfiguration.cs

@@ -0,0 +1,60 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ * 
+ *  http://aws.amazon.com/apache2.0
+ * 
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+/*
+ * Do not modify this file. This file is generated from the glue-2017-03-31.normal.json service model.
+ */
+using System;
+using System.Collections.Generic;
+using System.Xml.Serialization;
+using System.Text;
+using System.IO;
+using System.Net;
+
+using Amazon.Runtime;
+using Amazon.Runtime.Internal;
+
+#pragma warning disable CS0612,CS0618,CS1570
+namespace Amazon.Glue.Model
+{
+    /// <summary>
+    /// The configuration for a compaction optimizer. This configuration defines how data
+    /// files in your table will be compacted to improve query performance and reduce storage
+    /// costs.
+    /// </summary>
+    public partial class CompactionConfiguration
+    {
+        private IcebergCompactionConfiguration _icebergConfiguration;
+
+        /// <summary>
+        /// Gets and sets the property IcebergConfiguration. 
+        /// <para>
+        /// The configuration for an Iceberg compaction optimizer.
+        /// </para>
+        /// </summary>
+        public IcebergCompactionConfiguration IcebergConfiguration
+        {
+            get { return this._icebergConfiguration; }
+            set { this._icebergConfiguration = value; }
+        }
+
+        // Check to see if IcebergConfiguration property is set
+        internal bool IsSetIcebergConfiguration()
+        {
+            return this._icebergConfiguration != null;
+        }
+
+    }
+}

sdk/src/Services/Glue/Generated/Model/IcebergCompactionConfiguration.cs

@@ -0,0 +1,86 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ * 
+ *  http://aws.amazon.com/apache2.0
+ * 
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+/*
+ * Do not modify this file. This file is generated from the glue-2017-03-31.normal.json service model.
+ */
+using System;
+using System.Collections.Generic;
+using System.Xml.Serialization;
+using System.Text;
+using System.IO;
+using System.Net;
+
+using Amazon.Runtime;
+using Amazon.Runtime.Internal;
+
+#pragma warning disable CS0612,CS0618,CS1570
+namespace Amazon.Glue.Model
+{
+    /// <summary>
+    /// The configuration for an Iceberg compaction optimizer. This configuration defines
+    /// parameters for optimizing the layout of data files in Iceberg tables.
+    /// </summary>
+    public partial class IcebergCompactionConfiguration
+    {
+        private CompactionStrategy _strategy;
+
+        /// <summary>
+        /// Gets and sets the property Strategy. 
+        /// <para>
+        /// The strategy to use for compaction. Valid values are:
+        /// </para>
+        ///  <ul> <li> 
+        /// <para>
+        ///  <c>binpack</c>: Combines small files into larger files, typically targeting sizes
+        /// over 100MB, while applying any pending deletes. This is the recommended compaction
+        /// strategy for most use cases. 
+        /// </para>
+        ///  </li> <li> 
+        /// <para>
+        ///  <c>sort</c>: Organizes data based on specified columns which are sorted hierarchically
+        /// during compaction, improving query performance for filtered operations. This strategy
+        /// is recommended when your queries frequently filter on specific columns. To use this
+        /// strategy, you must first define a sort order in your Iceberg table properties using
+        /// the <c>sort_order</c> table property.
+        /// </para>
+        ///  </li> <li> 
+        /// <para>
+        ///  <c>z-order</c>: Optimizes data organization by blending multiple attributes into
+        /// a single scalar value that can be used for sorting, allowing efficient querying across
+        /// multiple dimensions. This strategy is recommended when you need to query data across
+        /// multiple dimensions simultaneously. To use this strategy, you must first define a
+        /// sort order in your Iceberg table properties using the <c>sort_order</c> table property.
+        /// 
+        /// </para>
+        ///  </li> </ul> 
+        /// <para>
+        /// If an input is not provided, the default value 'binpack' will be used.
+        /// </para>
+        /// </summary>
+        public CompactionStrategy Strategy
+        {
+            get { return this._strategy; }
+            set { this._strategy = value; }
+        }
+
+        // Check to see if Strategy property is set
+        internal bool IsSetStrategy()
+        {
+            return this._strategy != null;
+        }
+
+    }
+}

sdk/src/Services/Glue/Generated/Model/Internal/MarshallTransformations/CompactionConfigurationMarshaller.cs

@@ -0,0 +1,70 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ * 
+ *  http://aws.amazon.com/apache2.0
+ * 
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+/*
+ * Do not modify this file. This file is generated from the glue-2017-03-31.normal.json service model.
+ */
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Text;
+using System.Xml.Serialization;
+
+using Amazon.Glue.Model;
+using Amazon.Runtime;
+using Amazon.Runtime.Internal;
+using Amazon.Runtime.Internal.Transform;
+using Amazon.Runtime.Internal.Util;
+using ThirdParty.Json.LitJson;
+
+#pragma warning disable CS0612,CS0618
+namespace Amazon.Glue.Model.Internal.MarshallTransformations
+{
+    /// <summary>
+    /// CompactionConfiguration Marshaller
+    /// </summary>
+    public class CompactionConfigurationMarshaller : IRequestMarshaller<CompactionConfiguration, JsonMarshallerContext> 
+    {
+        /// <summary>
+        /// Unmarshaller the response from the service to the response class.
+        /// </summary>  
+        /// <param name="requestObject"></param>
+        /// <param name="context"></param>
+        /// <returns></returns>
+        public void Marshall(CompactionConfiguration requestObject, JsonMarshallerContext context)
+        {
+            if(requestObject == null)
+                return;
+            if(requestObject.IsSetIcebergConfiguration())
+            {
+                context.Writer.WritePropertyName("icebergConfiguration");
+                context.Writer.WriteObjectStart();
+
+                var marshaller = IcebergCompactionConfigurationMarshaller.Instance;
+                marshaller.Marshall(requestObject.IcebergConfiguration, context);
+
+                context.Writer.WriteObjectEnd();
+            }
+
+        }
+
+        /// <summary>
+        /// Singleton Marshaller.
+        /// </summary>
+        public readonly static CompactionConfigurationMarshaller Instance = new CompactionConfigurationMarshaller();
+
+    }
+}