[improve][broker] Upgrade bookkeeper to 4.17.2/commons-configuration to 2.x/grpc to 1.72.0 and enable ZooKeeper client to establish connection in read-only mode (#24468)

(cherry picked from commit 8d1ec6d)

Author: StevenLuMT

distribution/server/src/assemble/LICENSE.bin.txt

@@ -262,7 +262,7 @@ The Apache Software License, Version 2.0
      - com.fasterxml.jackson.module-jackson-module-parameter-names-2.14.2.jar
  * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar
  * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar
- * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.17.0.jar
+ * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.51.0.jar
  * Bitbucket -- org.bitbucket.b_c-jose4j-0.9.4.jar
  * Gson
     - com.google.code.gson-gson-2.8.9.jar
@@ -281,14 +281,17 @@ The Apache Software License, Version 2.0
     - com.yahoo.datasketches-memory-0.8.3.jar
     - com.yahoo.datasketches-sketches-core-0.8.3.jar
  * Apache Commons
+    - commons-beanutils-commons-beanutils-1.11.0.jar
     - commons-cli-commons-cli-1.9.0.jar
     - commons-codec-commons-codec-1.18.0.jar
     - commons-configuration-commons-configuration-1.10.jar
     - commons-io-commons-io-2.19.0.jar
     - commons-lang-commons-lang-2.6.jar
     - commons-logging-commons-logging-1.1.1.jar
+    - commons-collections-commons-collections-3.2.2.jar
     - org.apache.commons-commons-collections4-4.4.jar
     - org.apache.commons-commons-compress-1.27.1.jar
+    - org.apache.commons-commons-configuration2-2.12.0.jar
     - org.apache.commons-commons-lang3-3.17.0.jar
     - org.apache.commons-commons-text-1.13.1.jar
  * Netty
@@ -355,34 +358,34 @@ The Apache Software License, Version 2.0
     - net.java.dev.jna-jna-jpms-5.12.1.jar
     - net.java.dev.jna-jna-platform-jpms-5.12.1.jar
  * BookKeeper
-    - org.apache.bookkeeper-bookkeeper-common-4.17.1.jar
-    - org.apache.bookkeeper-bookkeeper-common-allocator-4.17.1.jar
-    - org.apache.bookkeeper-bookkeeper-proto-4.17.1.jar
-    - org.apache.bookkeeper-bookkeeper-server-4.17.1.jar
-    - org.apache.bookkeeper-bookkeeper-tools-framework-4.17.1.jar
-    - org.apache.bookkeeper-circe-checksum-4.17.1.jar
-    - org.apache.bookkeeper-cpu-affinity-4.17.1.jar
-    - org.apache.bookkeeper-statelib-4.17.1.jar
-    - org.apache.bookkeeper-stream-storage-api-4.17.1.jar
-    - org.apache.bookkeeper-stream-storage-common-4.17.1.jar
-    - org.apache.bookkeeper-stream-storage-java-client-4.17.1.jar
-    - org.apache.bookkeeper-stream-storage-java-client-base-4.17.1.jar
-    - org.apache.bookkeeper-stream-storage-proto-4.17.1.jar
-    - org.apache.bookkeeper-stream-storage-server-4.17.1.jar
-    - org.apache.bookkeeper-stream-storage-service-api-4.17.1.jar
-    - org.apache.bookkeeper-stream-storage-service-impl-4.17.1.jar
-    - org.apache.bookkeeper.http-http-server-4.17.1.jar
-    - org.apache.bookkeeper.http-vertx-http-server-4.17.1.jar
-    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.17.1.jar
-    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.17.1.jar
-    - org.apache.distributedlog-distributedlog-common-4.17.1.jar
-    - org.apache.distributedlog-distributedlog-core-4.17.1-tests.jar
-    - org.apache.distributedlog-distributedlog-core-4.17.1.jar
-    - org.apache.distributedlog-distributedlog-protocol-4.17.1.jar
-    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.17.1.jar
-    - org.apache.bookkeeper-bookkeeper-slogger-api-4.17.1.jar
-    - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.17.1.jar
-    - org.apache.bookkeeper-native-io-4.17.1.jar
+    - org.apache.bookkeeper-bookkeeper-common-4.17.2.jar
+    - org.apache.bookkeeper-bookkeeper-common-allocator-4.17.2.jar
+    - org.apache.bookkeeper-bookkeeper-proto-4.17.2.jar
+    - org.apache.bookkeeper-bookkeeper-server-4.17.2.jar
+    - org.apache.bookkeeper-bookkeeper-tools-framework-4.17.2.jar
+    - org.apache.bookkeeper-circe-checksum-4.17.2.jar
+    - org.apache.bookkeeper-cpu-affinity-4.17.2.jar
+    - org.apache.bookkeeper-statelib-4.17.2.jar
+    - org.apache.bookkeeper-stream-storage-api-4.17.2.jar
+    - org.apache.bookkeeper-stream-storage-common-4.17.2.jar
+    - org.apache.bookkeeper-stream-storage-java-client-4.17.2.jar
+    - org.apache.bookkeeper-stream-storage-java-client-base-4.17.2.jar
+    - org.apache.bookkeeper-stream-storage-proto-4.17.2.jar
+    - org.apache.bookkeeper-stream-storage-server-4.17.2.jar
+    - org.apache.bookkeeper-stream-storage-service-api-4.17.2.jar
+    - org.apache.bookkeeper-stream-storage-service-impl-4.17.2.jar
+    - org.apache.bookkeeper.http-http-server-4.17.2.jar
+    - org.apache.bookkeeper.http-vertx-http-server-4.17.2.jar
+    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.17.2.jar
+    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.17.2.jar
+    - org.apache.distributedlog-distributedlog-common-4.17.2.jar
+    - org.apache.distributedlog-distributedlog-core-4.17.2-tests.jar
+    - org.apache.distributedlog-distributedlog-core-4.17.2.jar
+    - org.apache.distributedlog-distributedlog-protocol-4.17.2.jar
+    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.17.2.jar
+    - org.apache.bookkeeper-bookkeeper-slogger-api-4.17.2.jar
+    - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.17.2.jar
+    - org.apache.bookkeeper-native-io-4.17.2.jar
   * Apache HTTP Client
     - org.apache.httpcomponents-httpclient-4.5.13.jar
     - org.apache.httpcomponents-httpcore-4.4.15.jar
@@ -429,29 +432,31 @@ The Apache Software License, Version 2.0
      - org.jetbrains.kotlin-kotlin-stdlib-jdk8-1.8.20.jar
      - org.jetbrains-annotations-13.0.jar
  * gRPC
-    - io.grpc-grpc-all-1.56.1.jar
-    - io.grpc-grpc-auth-1.56.1.jar
-    - io.grpc-grpc-context-1.56.1.jar
-    - io.grpc-grpc-core-1.56.1.jar
-    - io.grpc-grpc-protobuf-1.56.1.jar
-    - io.grpc-grpc-protobuf-lite-1.56.1.jar
-    - io.grpc-grpc-stub-1.56.1.jar
-    - io.grpc-grpc-alts-1.56.1.jar
-    - io.grpc-grpc-api-1.56.1.jar
-    - io.grpc-grpc-grpclb-1.56.1.jar
-    - io.grpc-grpc-netty-shaded-1.56.1.jar
-    - io.grpc-grpc-services-1.56.1.jar
-    - io.grpc-grpc-xds-1.56.1.jar
-    - io.grpc-grpc-rls-1.56.1.jar
-    - io.grpc-grpc-servlet-1.56.1.jar
-    - io.grpc-grpc-servlet-jakarta-1.56.1.jar
+    - io.grpc-grpc-all-1.72.0.jar
+    - io.grpc-grpc-auth-1.72.0.jar
+    - io.grpc-grpc-context-1.72.0.jar
+    - io.grpc-grpc-core-1.72.0.jar
+    - io.grpc-grpc-protobuf-1.72.0.jar
+    - io.grpc-grpc-protobuf-lite-1.72.0.jar
+    - io.grpc-grpc-stub-1.72.0.jar
+    - io.grpc-grpc-alts-1.72.0.jar
+    - io.grpc-grpc-api-1.72.0.jar
+    - io.grpc-grpc-grpclb-1.72.0.jar
+    - io.grpc-grpc-netty-shaded-1.72.0.jar
+    - io.grpc-grpc-services-1.72.0.jar
+    - io.grpc-grpc-xds-1.72.0.jar
+    - io.grpc-grpc-rls-1.72.0.jar
+    - io.grpc-grpc-servlet-1.72.0.jar
+    - io.grpc-grpc-servlet-jakarta-1.72.0.jar
     - io.grpc-grpc-util-1.60.0.jar
+    - io.grpc-grpc-opentelemetry-1.72.0.jar
+    - io.grpc-grpc-gcp-csm-observability-1.72.0.jar
+    - io.grpc-grpc-inprocess-1.72.0.jar
   * Perfmark
     - io.perfmark-perfmark-api-0.26.0.jar
   * OpenCensus
     - io.opencensus-opencensus-api-0.28.0.jar
     - io.opencensus-opencensus-contrib-http-util-0.28.0.jar
-    - io.opencensus-opencensus-proto-0.2.0.jar
   * Jodah
     - net.jodah-typetools-0.5.0.jar
     - dev.failsafe-failsafe-3.3.2.jar
@@ -505,7 +510,7 @@ The Apache Software License, Version 2.0
   * Google HTTP Client
     - com.google.http-client-google-http-client-gson-1.41.0.jar
     - com.google.http-client-google-http-client-1.41.0.jar
-    - com.google.auto.value-auto-value-annotations-1.10.1.jar
+    - com.google.auto.value-auto-value-annotations-1.11.0.jar
     - com.google.re2j-re2j-1.7.jar
   * Jetcd - shaded
   * IPAddress
@@ -536,15 +541,17 @@ The Apache Software License, Version 2.0
     - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java17-1.33.3-alpha.jar
     - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java8-1.33.3-alpha.jar
     - io.opentelemetry.semconv-opentelemetry-semconv-1.29.0-alpha.jar
+    - com.google.cloud.opentelemetry-detector-resources-support-0.33.0.jar
+    - io.opentelemetry.contrib-opentelemetry-gcp-resources-1.43.0-alpha.jar
   * Spotify completable-futures
     - com.spotify-completable-futures-0.3.6.jar
   * JSpecify
     - org.jspecify-jspecify-1.0.0.jar
 
 BSD 3-clause "New" or "Revised" License
  * Google auth library
-    - com.google.auth-google-auth-library-credentials-1.4.0.jar -- ../licenses/LICENSE-google-auth-library.txt
-    - com.google.auth-google-auth-library-oauth2-http-1.4.0.jar -- ../licenses/LICENSE-google-auth-library.txt
+    - com.google.auth-google-auth-library-credentials-1.24.1.jar -- ../licenses/LICENSE-google-auth-library.txt
+    - com.google.auth-google-auth-library-oauth2-http-1.24.1.jar -- ../licenses/LICENSE-google-auth-library.txt
  * LevelDB -- (included in org.rocksdb.*.jar) -- ../licenses/LICENSE-LevelDB.txt
  * JSR305 -- com.google.code.findbugs-jsr305-3.0.2.jar -- ../licenses/LICENSE-JSR305.txt
  * JLine -- jline-jline-2.14.6.jar -- ../licenses/LICENSE-JLine.txt

distribution/shell/src/assemble/LICENSE.bin.txt

@@ -339,13 +339,14 @@ The Apache Software License, Version 2.0
     - sketches-core-0.8.3.jar
  * Apache Commons
     - commons-codec-1.18.0.jar
-    - commons-configuration-1.10.jar
     - commons-io-2.19.0.jar
-    - commons-lang-2.6.jar
     - commons-logging-1.2.jar
     - commons-lang3-3.17.0.jar
     - commons-text-1.13.1.jar
     - commons-compress-1.27.1.jar
+    - commons-beanutils-1.11.0.jar
+    - commons-collections-3.2.2.jar
+    - commons-configuration2-2.12.0.jar
  * Netty
     - netty-buffer-4.1.122.Final.jar
     - netty-codec-4.1.122.Final.jar
@@ -393,9 +394,9 @@ The Apache Software License, Version 2.0
     - opentelemetry-context-1.45.0.jar
 
  * BookKeeper
-    - bookkeeper-common-allocator-4.17.1.jar
-    - cpu-affinity-4.17.1.jar
-    - circe-checksum-4.17.1.jar
+    - bookkeeper-common-allocator-4.17.2.jar
+    - cpu-affinity-4.17.2.jar
+    - circe-checksum-4.17.2.jar
   * AirCompressor
      - aircompressor-0.27.jar
  * AsyncHttpClient

pom.xml

@@ -137,7 +137,7 @@ flexible messaging model and an intuitive client API.</description>
     <!-- apache commons -->
     <commons-compress.version>1.27.1</commons-compress.version>
 
-    <bookkeeper.version>4.17.1</bookkeeper.version>
+    <bookkeeper.version>4.17.2</bookkeeper.version>
     <zookeeper.version>3.9.3</zookeeper.version>
     <commons-cli.version>1.9.0</commons-cli.version>
     <commons-text.version>1.13.1</commons-text.version>
@@ -170,7 +170,7 @@ flexible messaging model and an intuitive client API.</description>
     <zt-zip.version>1.17</zt-zip.version>
     <protobuf3.version>3.25.5</protobuf3.version>
     <protoc3.version>${protobuf3.version}</protoc3.version>
-    <grpc.version>1.56.1</grpc.version>
+    <grpc.version>1.72.0</grpc.version>
     <google-http-client.version>1.41.0</google-http-client.version>
     <perfmark.version>0.26.0</perfmark.version>
     <protoc-gen-grpc-java.version>${grpc.version}</protoc-gen-grpc-java.version>
@@ -263,6 +263,7 @@ flexible messaging model and an intuitive client API.</description>
     <opentelemetry.instrumentation.alpha.version>${opentelemetry.instrumentation.version}-alpha</opentelemetry.instrumentation.alpha.version>
     <opentelemetry.semconv.version>1.29.0-alpha</opentelemetry.semconv.version>
     <picocli.version>4.7.5</picocli.version>
+    <re2j.version>1.7</re2j.version>
     <completable-futures.version>0.3.6</completable-futures.version>
     <failsafe.version>3.3.2</failsafe.version>
 
@@ -662,6 +663,12 @@ flexible messaging model and an intuitive client API.</description>
         <version>${bookkeeper.version}</version>
       </dependency>
 
+      <dependency>
+        <groupId>com.google.re2j</groupId>
+        <artifactId>re2j</artifactId>
+        <version>${re2j.version}</version>
+      </dependency>
+
       <dependency>
         <groupId>com.spotify</groupId>
         <artifactId>completable-futures</artifactId>

pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java

@@ -40,7 +40,7 @@
 import org.apache.bookkeeper.net.BookieNode;
 import org.apache.bookkeeper.net.BookieSocketAddress;
 import org.apache.bookkeeper.proto.BookieAddressResolver;
-import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration2.Configuration;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.common.policies.data.BookieInfo;
 import org.apache.pulsar.common.policies.data.BookiesRackConfiguration;

pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java

@@ -43,7 +43,7 @@
 import org.apache.bookkeeper.replication.AutoRecoveryMain;
 import org.apache.bookkeeper.server.conf.BookieConfiguration;
 import org.apache.bookkeeper.stats.StatsProvider;
-import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration2.ex.ConfigurationException;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.core.util.datetime.FixedDateFormat;
 import org.apache.pulsar.broker.PulsarServerException;

pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java

@@ -29,6 +29,7 @@
 import org.apache.bookkeeper.stream.storage.api.cluster.ClusterInitializer;
 import org.apache.bookkeeper.stream.storage.impl.cluster.ZkClusterInitializer;
 import org.apache.bookkeeper.util.BookKeeperConstants;
+import org.apache.commons.configuration2.convert.DisabledListDelimiterHandler;
 import org.apache.pulsar.bookie.rackawareness.BookieRackAffinityMapping;
 import org.apache.pulsar.broker.resources.NamespaceResources;
 import org.apache.pulsar.broker.resources.PulsarResources;
@@ -304,7 +305,7 @@ private static void initializeCluster(Arguments arguments, int bundleNumberForDe
         // Format BookKeeper ledger storage metadata
         if (arguments.existingBkMetadataServiceUri == null && arguments.bookieMetadataServiceUri == null) {
             ServerConfiguration bkConf = new ServerConfiguration();
-            bkConf.setDelimiterParsingDisabled(true);
+            bkConf.setListDelimiterHandler(new DisabledListDelimiterHandler());
             bkConf.setMetadataServiceUri("metadata-store:" + arguments.metadataStoreUrl);
             bkConf.setZkTimeout(arguments.zkSessionTimeoutMillis);
             // only format if /ledgers doesn't exist

pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java

@@ -36,7 +36,7 @@
 import org.apache.bookkeeper.stats.NullStatsProvider;
 import org.apache.bookkeeper.stats.StatsLogger;
 import org.apache.bookkeeper.stats.StatsProvider;
-import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration2.Configuration;
 import org.apache.pulsar.broker.stats.prometheus.metrics.PrometheusMetricsProvider;
 import org.apache.pulsar.broker.storage.ManagedLedgerStorage;
 import org.apache.pulsar.common.policies.data.EnsemblePlacementPolicyConfig;

pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusMetricsProvider.java

@@ -31,7 +31,7 @@
 import org.apache.bookkeeper.stats.CachingStatsProvider;
 import org.apache.bookkeeper.stats.StatsLogger;
 import org.apache.bookkeeper.stats.StatsProvider;
-import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration2.Configuration;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.client.util.ExecutorProvider;
 

pulsar-broker/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java

@@ -67,7 +67,7 @@
 import org.apache.bookkeeper.stream.server.StreamStorageLifecycleComponent;
 import org.apache.bookkeeper.stream.storage.api.cluster.ClusterInitializer;
 import org.apache.bookkeeper.stream.storage.impl.cluster.ZkClusterInitializer;
-import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration2.CompositeConfiguration;
 import org.apache.commons.io.FileUtils;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;

pulsar-broker/src/test/java/org/apache/pulsar/broker/BookKeeperClientFactoryImplTest.java

@@ -36,7 +36,7 @@
 import org.apache.bookkeeper.conf.ClientConfiguration;
 import org.apache.bookkeeper.net.CachedDNSToSwitchMapping;
 import org.apache.bookkeeper.stats.StatsLogger;
-import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration2.ex.ConfigurationException;
 import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.pulsar.bookie.rackawareness.BookieRackAffinityMapping;
 import org.apache.pulsar.metadata.api.MetadataStore;