Skip to content

Commit f0164c0

Browse files
author
AWS
committed
Amazon Kinesis Firehose Update: This release 1) Add configurable buffering hints for Snowflake as destination. 2) Add ReadFromTimestamp for MSK As Source. Firehose will start reading data from MSK Cluster using offset associated with this timestamp. 3) Gated public beta release to add Apache Iceberg tables as destination.
1 parent e57fa9b commit f0164c0

File tree

2 files changed

+202
-0
lines changed

2 files changed

+202
-0
lines changed

‎.changes/next-release/feature-AmazonKinesisFirehose-014573f.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"type": "feature",
3+
"category": "Amazon Kinesis Firehose",
4+
"contributor": "",
5+
"description": "This release 1) Add configurable buffering hints for Snowflake as destination. 2) Add ReadFromTimestamp for MSK As Source. Firehose will start reading data from MSK Cluster using offset associated with this timestamp. 3) Gated public beta release to add Apache Iceberg tables as destination."
6+
}

‎services/firehose/src/main/resources/codegen-resources/service-2.json

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -639,6 +639,16 @@
639639
},
640640
"documentation":"<p>Describes hints for the buffering to perform before delivering data to the destination. These options are treated as hints, and therefore Firehose might choose to use different values when it is optimal. The <code>SizeInMBs</code> and <code>IntervalInSeconds</code> parameters are optional. However, if specify a value for one of them, you must also provide a value for the other.</p>"
641641
},
642+
"CatalogConfiguration":{
643+
"type":"structure",
644+
"members":{
645+
"CatalogARN":{
646+
"shape":"GlueDataCatalogARN",
647+
"documentation":"<p> Specifies the Glue catalog ARN indentifier of the destination Apache Iceberg Tables. You must specify the ARN in the format <code>arn:aws:glue:region:account-id:catalog</code>. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
648+
}
649+
},
650+
"documentation":"<p> Describes the containers where the destination Apache Iceberg Tables are persisted. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
651+
},
642652
"CloudWatchLoggingOptions":{
643653
"type":"structure",
644654
"members":{
@@ -789,6 +799,10 @@
789799
"SnowflakeDestinationConfiguration":{
790800
"shape":"SnowflakeDestinationConfiguration",
791801
"documentation":"<p>Configure Snowflake destination</p>"
802+
},
803+
"IcebergDestinationConfiguration":{
804+
"shape":"IcebergDestinationConfiguration",
805+
"documentation":"<p> Configure Apache Iceberg Tables destination. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
792806
}
793807
}
794808
},
@@ -1133,6 +1147,10 @@
11331147
"AmazonOpenSearchServerlessDestinationDescription":{
11341148
"shape":"AmazonOpenSearchServerlessDestinationDescription",
11351149
"documentation":"<p>The destination in the Serverless offering for Amazon OpenSearch Service.</p>"
1150+
},
1151+
"IcebergDestinationDescription":{
1152+
"shape":"IcebergDestinationDescription",
1153+
"documentation":"<p> Describes a destination in Apache Iceberg Tables. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
11361154
}
11371155
},
11381156
"documentation":"<p>Describes the destination for a delivery stream.</p>"
@@ -1147,6 +1165,36 @@
11471165
"min":1,
11481166
"pattern":"[a-zA-Z0-9-]+"
11491167
},
1168+
"DestinationTableConfiguration":{
1169+
"type":"structure",
1170+
"required":[
1171+
"DestinationTableName",
1172+
"DestinationDatabaseName"
1173+
],
1174+
"members":{
1175+
"DestinationTableName":{
1176+
"shape":"NonEmptyStringWithoutWhitespace",
1177+
"documentation":"<p> Specifies the name of the Apache Iceberg Table. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
1178+
},
1179+
"DestinationDatabaseName":{
1180+
"shape":"NonEmptyStringWithoutWhitespace",
1181+
"documentation":"<p> The name of the Apache Iceberg database. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
1182+
},
1183+
"UniqueKeys":{
1184+
"shape":"ListOfNonEmptyStringsWithoutWhitespace",
1185+
"documentation":"<p> A list of unique keys for a given Apache Iceberg table. Firehose will use these for running Create/Update/Delete operations on the given Iceberg table. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
1186+
},
1187+
"S3ErrorOutputPrefix":{
1188+
"shape":"ErrorOutputPrefix",
1189+
"documentation":"<p> The table specific S3 error output prefix. All the errors that occurred while delivering to this table will be prefixed with this value in S3 destination. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
1190+
}
1191+
},
1192+
"documentation":"<p> Describes the configuration of a destination in Apache Iceberg Tables. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
1193+
},
1194+
"DestinationTableConfigurationList":{
1195+
"type":"list",
1196+
"member":{"shape":"DestinationTableConfiguration"}
1197+
},
11501198
"DocumentIdOptions":{
11511199
"type":"structure",
11521200
"required":["DefaultDocumentIdFormat"],
@@ -1690,6 +1738,12 @@
16901738
"min":0,
16911739
"pattern":"^$|\\.[0-9a-z!\\-_.*'()]+"
16921740
},
1741+
"GlueDataCatalogARN":{
1742+
"type":"string",
1743+
"max":512,
1744+
"min":1,
1745+
"pattern":"arn:.*"
1746+
},
16931747
"HECAcknowledgmentTimeoutInSeconds":{
16941748
"type":"integer",
16951749
"max":600,
@@ -1990,6 +2044,99 @@
19902044
"pattern":"https://.*",
19912045
"sensitive":true
19922046
},
2047+
"IcebergDestinationConfiguration":{
2048+
"type":"structure",
2049+
"required":[
2050+
"RoleARN",
2051+
"CatalogConfiguration",
2052+
"S3Configuration"
2053+
],
2054+
"members":{
2055+
"DestinationTableConfigurationList":{
2056+
"shape":"DestinationTableConfigurationList",
2057+
"documentation":"<p> Provides a list of <code>DestinationTableConfigurations</code> which Firehose uses to deliver data to Apache Iceberg tables. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
2058+
},
2059+
"BufferingHints":{"shape":"BufferingHints"},
2060+
"CloudWatchLoggingOptions":{"shape":"CloudWatchLoggingOptions"},
2061+
"ProcessingConfiguration":{"shape":"ProcessingConfiguration"},
2062+
"S3BackupMode":{
2063+
"shape":"IcebergS3BackupMode",
2064+
"documentation":"<p> Describes how Firehose will backup records. Currently,Firehose only supports <code>FailedDataOnly</code> for preview. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
2065+
},
2066+
"RetryOptions":{"shape":"RetryOptions"},
2067+
"RoleARN":{
2068+
"shape":"RoleARN",
2069+
"documentation":"<p> The Amazon Resource Name (ARN) of the Apache Iceberg tables role. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
2070+
},
2071+
"CatalogConfiguration":{
2072+
"shape":"CatalogConfiguration",
2073+
"documentation":"<p> Configuration describing where the destination Apache Iceberg Tables are persisted. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
2074+
},
2075+
"S3Configuration":{"shape":"S3DestinationConfiguration"}
2076+
},
2077+
"documentation":"<p> Specifies the destination configure settings for Apache Iceberg Table. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
2078+
},
2079+
"IcebergDestinationDescription":{
2080+
"type":"structure",
2081+
"members":{
2082+
"DestinationTableConfigurationList":{
2083+
"shape":"DestinationTableConfigurationList",
2084+
"documentation":"<p> Provides a list of <code>DestinationTableConfigurations</code> which Firehose uses to deliver data to Apache Iceberg tables. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
2085+
},
2086+
"BufferingHints":{"shape":"BufferingHints"},
2087+
"CloudWatchLoggingOptions":{"shape":"CloudWatchLoggingOptions"},
2088+
"ProcessingConfiguration":{"shape":"ProcessingConfiguration"},
2089+
"S3BackupMode":{
2090+
"shape":"IcebergS3BackupMode",
2091+
"documentation":"<p> Describes how Firehose will backup records. Currently,Firehose only supports <code>FailedDataOnly</code> for preview. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
2092+
},
2093+
"RetryOptions":{"shape":"RetryOptions"},
2094+
"RoleARN":{
2095+
"shape":"RoleARN",
2096+
"documentation":"<p> The Amazon Resource Name (ARN) of the Apache Iceberg Tables role. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
2097+
},
2098+
"CatalogConfiguration":{
2099+
"shape":"CatalogConfiguration",
2100+
"documentation":"<p> Configuration describing where the destination Iceberg tables are persisted. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
2101+
},
2102+
"S3DestinationDescription":{"shape":"S3DestinationDescription"}
2103+
},
2104+
"documentation":"<p> Describes a destination in Apache Iceberg Tables. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
2105+
},
2106+
"IcebergDestinationUpdate":{
2107+
"type":"structure",
2108+
"members":{
2109+
"DestinationTableConfigurationList":{
2110+
"shape":"DestinationTableConfigurationList",
2111+
"documentation":"<p> Provides a list of <code>DestinationTableConfigurations</code> which Firehose uses to deliver data to Apache Iceberg tables. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
2112+
},
2113+
"BufferingHints":{"shape":"BufferingHints"},
2114+
"CloudWatchLoggingOptions":{"shape":"CloudWatchLoggingOptions"},
2115+
"ProcessingConfiguration":{"shape":"ProcessingConfiguration"},
2116+
"S3BackupMode":{
2117+
"shape":"IcebergS3BackupMode",
2118+
"documentation":"<p> Describes how Firehose will backup records. Currently,Firehose only supports <code>FailedDataOnly</code> for preview. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
2119+
},
2120+
"RetryOptions":{"shape":"RetryOptions"},
2121+
"RoleARN":{
2122+
"shape":"RoleARN",
2123+
"documentation":"<p> The Amazon Resource Name (ARN) of the Apache Iceberg Tables role. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
2124+
},
2125+
"CatalogConfiguration":{
2126+
"shape":"CatalogConfiguration",
2127+
"documentation":"<p> Configuration describing where the destination Iceberg tables are persisted. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
2128+
},
2129+
"S3Configuration":{"shape":"S3DestinationConfiguration"}
2130+
},
2131+
"documentation":"<p> Describes an update for a destination in Apache Iceberg Tables. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
2132+
},
2133+
"IcebergS3BackupMode":{
2134+
"type":"string",
2135+
"enum":[
2136+
"FailedDataOnly",
2137+
"AllData"
2138+
]
2139+
},
19932140
"InputFormatConfiguration":{
19942141
"type":"structure",
19952142
"members":{
@@ -2235,6 +2382,10 @@
22352382
"AuthenticationConfiguration":{
22362383
"shape":"AuthenticationConfiguration",
22372384
"documentation":"<p>The authentication configuration of the Amazon MSK cluster.</p>"
2385+
},
2386+
"ReadFromTimestamp":{
2387+
"shape":"ReadFromTimestamp",
2388+
"documentation":"<p>The start date and time in UTC for the offset position within your MSK topic from where Firehose begins to read. By default, this is set to timestamp when Firehose becomes Active. </p> <p>If you want to create a Firehose stream with Earliest start position from SDK or CLI, you need to set the <code>ReadFromTimestamp</code> parameter to Epoch (1970-01-01T00:00:00Z). </p>"
22382389
}
22392390
},
22402391
"documentation":"<p>The configuration for the Amazon MSK cluster to be used as the source for a delivery stream.</p>"
@@ -2257,6 +2408,10 @@
22572408
"DeliveryStartTimestamp":{
22582409
"shape":"DeliveryStartTimestamp",
22592410
"documentation":"<p>Firehose starts retrieving records from the topic within the Amazon MSK cluster starting with this timestamp.</p>"
2411+
},
2412+
"ReadFromTimestamp":{
2413+
"shape":"ReadFromTimestamp",
2414+
"documentation":"<p>The start date and time in UTC for the offset position within your MSK topic from where Firehose begins to read. By default, this is set to timestamp when Firehose becomes Active. </p> <p>If you want to create a Firehose stream with Earliest start position from SDK or CLI, you need to set the <code>ReadFromTimestampUTC</code> parameter to Epoch (1970-01-01T00:00:00Z). </p>"
22602415
}
22612416
},
22622417
"documentation":"<p>Details about the Amazon MSK cluster used as the source for a Firehose delivery stream.</p>"
@@ -2636,6 +2791,7 @@
26362791
"type":"string",
26372792
"min":1
26382793
},
2794+
"ReadFromTimestamp":{"type":"timestamp"},
26392795
"Record":{
26402796
"type":"structure",
26412797
"required":["Data"],
@@ -3113,6 +3269,30 @@
31133269
"pattern":".+?\\.snowflakecomputing\\.com",
31143270
"sensitive":true
31153271
},
3272+
"SnowflakeBufferingHints":{
3273+
"type":"structure",
3274+
"members":{
3275+
"SizeInMBs":{
3276+
"shape":"SnowflakeBufferingSizeInMBs",
3277+
"documentation":"<p> Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 1. </p>"
3278+
},
3279+
"IntervalInSeconds":{
3280+
"shape":"SnowflakeBufferingIntervalInSeconds",
3281+
"documentation":"<p> Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 0. </p>"
3282+
}
3283+
},
3284+
"documentation":"<p> Describes the buffering to perform before delivering data to the Snowflake destination. If you do not specify any value, Firehose uses the default values. </p>"
3285+
},
3286+
"SnowflakeBufferingIntervalInSeconds":{
3287+
"type":"integer",
3288+
"max":900,
3289+
"min":0
3290+
},
3291+
"SnowflakeBufferingSizeInMBs":{
3292+
"type":"integer",
3293+
"max":128,
3294+
"min":1
3295+
},
31163296
"SnowflakeContentColumnName":{
31173297
"type":"string",
31183298
"max":255,
@@ -3210,6 +3390,10 @@
32103390
"SecretsManagerConfiguration":{
32113391
"shape":"SecretsManagerConfiguration",
32123392
"documentation":"<p> The configuration that defines how you access secrets for Snowflake. </p>"
3393+
},
3394+
"BufferingHints":{
3395+
"shape":"SnowflakeBufferingHints",
3396+
"documentation":"<p> Describes the buffering to perform before delivering data to the Snowflake destination. If you do not specify any value, Firehose uses the default values. </p>"
32133397
}
32143398
},
32153399
"documentation":"<p>Configure Snowflake destination</p>"
@@ -3275,6 +3459,10 @@
32753459
"SecretsManagerConfiguration":{
32763460
"shape":"SecretsManagerConfiguration",
32773461
"documentation":"<p> The configuration that defines how you access secrets for Snowflake. </p>"
3462+
},
3463+
"BufferingHints":{
3464+
"shape":"SnowflakeBufferingHints",
3465+
"documentation":"<p> Describes the buffering to perform before delivering data to the Snowflake destination. If you do not specify any value, Firehose uses the default values. </p>"
32783466
}
32793467
},
32803468
"documentation":"<p>Optional Snowflake destination description</p>"
@@ -3344,6 +3532,10 @@
33443532
"SecretsManagerConfiguration":{
33453533
"shape":"SecretsManagerConfiguration",
33463534
"documentation":"<p> Describes the Secrets Manager configuration in Snowflake. </p>"
3535+
},
3536+
"BufferingHints":{
3537+
"shape":"SnowflakeBufferingHints",
3538+
"documentation":"<p> Describes the buffering to perform before delivering data to the Snowflake destination. </p>"
33473539
}
33483540
},
33493541
"documentation":"<p>Update to configuration settings</p>"
@@ -3846,6 +4038,10 @@
38464038
"SnowflakeDestinationUpdate":{
38474039
"shape":"SnowflakeDestinationUpdate",
38484040
"documentation":"<p>Update to the Snowflake destination configuration settings.</p>"
4041+
},
4042+
"IcebergDestinationUpdate":{
4043+
"shape":"IcebergDestinationUpdate",
4044+
"documentation":"<p> Describes an update for a destination in Apache Iceberg Tables. </p> <p>Amazon Data Firehose is in preview release and is subject to change.</p>"
38494045
}
38504046
}
38514047
},

0 commit comments

Comments
 (0)