[FLINK-36821] Update to Kafka 3.9.0

Signed-off-by: Thomas Cooper <[email protected]>

Author: tomncooper

flink-connector-kafka/archunit-violations/86dfd459-67a9-4b26-9b5c-0b0bbf22681a

@@ -2,4 +2,3 @@ org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema.open(org.ap
 org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema.serialize(java.lang.Object, org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema$KafkaSinkContext, java.lang.Long): Argument leaf type org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema$KafkaSinkContext does not satisfy: reside outside of package 'org.apache.flink..' or reside in any package ['..shaded..'] or annotated with @Public or annotated with @PublicEvolving or annotated with @Deprecated
 org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer.getPartitionOffsets(java.util.Collection, org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer$PartitionOffsetsRetriever): Argument leaf type org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer$PartitionOffsetsRetriever does not satisfy: reside outside of package 'org.apache.flink..' or reside in any package ['..shaded..'] or annotated with @Public or annotated with @PublicEvolving or annotated with @Deprecated
 org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer.invoke(org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer$KafkaTransactionState, java.lang.Object, org.apache.flink.streaming.api.functions.sink.SinkFunction$Context): Argument leaf type org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer$KafkaTransactionState does not satisfy: reside outside of package 'org.apache.flink..' or reside in any package ['..shaded..'] or annotated with @Public or annotated with @PublicEvolving or annotated with @Deprecated
-org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition.dropLeaderData(java.util.List): Argument leaf type org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartitionLeader does not satisfy: reside outside of package 'org.apache.flink..' or reside in any package ['..shaded..'] or annotated with @Public or annotated with @PublicEvolving or annotated with @Deprecated

flink-connector-kafka/archunit-violations/c0d94764-76a0-4c50-b617-70b1754c4612

@@ -37,6 +37,7 @@
 import org.apache.kafka.common.Node;
 import org.apache.kafka.common.PartitionInfo;
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.Uuid;
 import org.apache.kafka.common.errors.ProducerFencedException;
 import org.apache.kafka.common.requests.FindCoordinatorRequest;
 import org.slf4j.Logger;
@@ -156,6 +157,11 @@ public List<PartitionInfo> partitionsFor(String topic) {
         return kafkaProducer.metrics();
     }
 
+    @Override
+    public Uuid clientInstanceId(Duration timeout) {
+        return kafkaProducer.clientInstanceId(timeout);
+    }
+
     @Override
     public void close() {
         throw new UnsupportedOperationException(

flink-connector-kafka/archunit-violations/d853eb69-8c04-4246-9a5e-4f5911286b1d

@@ -24,9 +24,9 @@
  */
 public class DockerImageVersions {
 
-    public static final String KAFKA = "confluentinc/cp-kafka:7.4.4";
+    public static final String KAFKA = "confluentinc/cp-kafka:7.7.2";
 
-    public static final String SCHEMA_REGISTRY = "confluentinc/cp-schema-registry:7.4.4";
+    public static final String SCHEMA_REGISTRY = "confluentinc/cp-schema-registry:7.7.2";
 
-    public static final String ZOOKEEPER = "zookeeper:3.4.14";
+    public static final String ZOOKEEPER = "zookeeper:3.8.4";
 }

flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/FlinkKafkaInternalProducer.java

@@ -6,4 +6,4 @@ The Apache Software Foundation (http://www.apache.org/).
 
 This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt)
 
-- org.apache.kafka:kafka-clients:3.4.0
+- org.apache.kafka:kafka-clients:3.9.0

flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/testutils/DockerImageVersions.java

@@ -51,10 +51,10 @@ under the License.
 
     <properties>
         <flink.version>1.20.0</flink.version>
-        <kafka.version>3.4.0</kafka.version>
-        <confluent.version>7.4.4</confluent.version>
+        <kafka.version>3.9.0</kafka.version>
+        <confluent.version>7.7.1</confluent.version>
 
-        <jackson-bom.version>2.15.2</jackson-bom.version>
+        <jackson-bom.version>2.16.2</jackson-bom.version>
         <junit4.version>4.13.2</junit4.version>
         <junit5.version>5.9.1</junit5.version>
         <assertj.version>3.23.1</assertj.version>
@@ -65,16 +65,16 @@ under the License.
         <byte-buddy.version>1.12.10</byte-buddy.version>
         <commons-cli.version>1.5.0</commons-cli.version>
         <scala.binary.version>2.12</scala.binary.version>
-        <scala-reflect.version>2.12.7</scala-reflect.version>
-        <scala-library.version>2.12.7</scala-library.version>
+        <scala-reflect.version>2.12.19</scala-reflect.version>
+        <scala-library.version>2.12.19</scala-library.version>
         <snappy-java.version>1.1.10.5</snappy-java.version>
         <avro.version>1.11.4</avro.version>
         <guava.version>32.1.2-jre</guava.version>
 
         <japicmp.skip>false</japicmp.skip>
         <japicmp.referenceVersion>1.17.0</japicmp.referenceVersion>
 
-        <slf4j.version>1.7.32</slf4j.version>
+        <slf4j.version>1.7.36</slf4j.version>
         <log4j.version>2.17.1</log4j.version>
 
         <flink.parent.artifactId>flink-connector-kafka-parent</flink.parent.artifactId>