Upgrade to RocksDB 9.9.3

bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt

@@ -271,7 +271,7 @@ Apache Software License, Version 2.
 - lib/org.eclipse.jetty-jetty-servlet-9.4.55.v20240627.jar [22]
 - lib/org.eclipse.jetty-jetty-util-9.4.55.v20240627.jar [22]
 - lib/org.eclipse.jetty-jetty-util-ajax-9.4.55.v20240627.jar [22]
-- lib/org.rocksdb-rocksdbjni-7.10.2.jar [23]
+- lib/org.rocksdb-rocksdbjni-9.9.3.jar [23]
 - lib/com.beust-jcommander-1.82.jar [24]
 - lib/com.yahoo.datasketches-memory-0.8.3.jar [25]
 - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25]
@@ -378,7 +378,7 @@ Apache Software License, Version 2.
 [20] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6
 [21] Source available at https://github.com/apache/zookeeper/tree/release-3.8.0
 [22] Source available at https://github.com/eclipse/jetty.project/tree/jetty-9.4.48.v20220622
-[23] Source available at https://github.com/facebook/rocksdb/tree/v7.10.2
+[23] Source available at https://github.com/facebook/rocksdb/tree/v9.9.3
 [24] Source available at https://github.com/cbeust/jcommander/tree/1.82
 [25] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3
 [26] Source available at https://github.com/lz4/lz4-java/tree/1.3.0
@@ -639,7 +639,7 @@ This private header is also used by Apple's open source
     * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h
 
 ------------------------------------------------------------------------------------
-lib/org.rocksdb-rocksdbjni-7.10.2.jar is derived from leveldb, which is under the following license.
+lib/org.rocksdb-rocksdbjni-9.9.3.jar is derived from leveldb, which is under the following license.
 
 Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 

bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt

@@ -271,7 +271,7 @@ Apache Software License, Version 2.
 - lib/org.eclipse.jetty-jetty-servlet-9.4.55.v20240627.jar [22]
 - lib/org.eclipse.jetty-jetty-util-9.4.55.v20240627.jar [22]
 - lib/org.eclipse.jetty-jetty-util-ajax-9.4.55.v20240627.jar [22]
-- lib/org.rocksdb-rocksdbjni-7.10.2.jar [23]
+- lib/org.rocksdb-rocksdbjni-9.9.3.jar [23]
 - lib/com.beust-jcommander-1.82.jar [24]
 - lib/com.yahoo.datasketches-memory-0.8.3.jar [25]
 - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25]
@@ -374,7 +374,7 @@ Apache Software License, Version 2.
 [20] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6
 [21] Source available at https://github.com/apache/zookeeper/tree/release-3.8.0
 [22] Source available at https://github.com/eclipse/jetty.project/tree/jetty-9.4.48.v20220622
-[23] Source available at https://github.com/facebook/rocksdb/tree/v7.10.2
+[23] Source available at https://github.com/facebook/rocksdb/tree/v9.9.3
 [24] Source available at https://github.com/cbeust/jcommander/tree/1.82
 [25] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3
 [26] Source available at https://github.com/lz4/lz4-java/tree/1.3.0
@@ -634,7 +634,7 @@ This private header is also used by Apple's open source
     * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h
 
 ------------------------------------------------------------------------------------
-lib/org.rocksdb-rocksdbjni-7.10.2.jar is derived from leveldb, which is under the following license.
+lib/org.rocksdb-rocksdbjni-9.9.3.jar is derived from leveldb, which is under the following license.
 
 Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 

bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java

@@ -48,6 +48,7 @@
 import org.rocksdb.ColumnFamilyDescriptor;
 import org.rocksdb.ColumnFamilyHandle;
 import org.rocksdb.CompressionType;
+import org.rocksdb.ConfigOptions;
 import org.rocksdb.DBOptions;
 import org.rocksdb.Env;
 import org.rocksdb.InfoLogLevel;
@@ -155,8 +156,10 @@ private RocksDB initializeRocksDBWithConfFile(String basePath, String subPath, D
         DBOptions dbOptions = new DBOptions();
         final List<ColumnFamilyDescriptor> cfDescs = new ArrayList<>();
         final List<ColumnFamilyHandle> cfHandles = new ArrayList<>();
-        try {
-            OptionsUtil.loadOptionsFromFile(dbFilePath, Env.getDefault(), dbOptions, cfDescs, false);
+        try (final ConfigOptions cfgOpts = new ConfigOptions()
+                .setIgnoreUnknownOptions(false)
+                .setEnv(Env.getDefault())) {
+            OptionsUtil.loadOptionsFromFile(cfgOpts, dbFilePath, dbOptions, cfDescs);
             // Configure file path
             String logPath = conf.getString(ROCKSDB_LOG_PATH, "");
             if (!logPath.isEmpty()) {

pom.xml

@@ -167,7 +167,7 @@
     <protoc3.version>${protobuf.version}</protoc3.version>
     <protoc-gen-grpc-java.version>${grpc.version}</protoc-gen-grpc-java.version>
     <reflections.version>0.9.11</reflections.version>
-    <rocksdb.version>7.10.2</rocksdb.version>
+    <rocksdb.version>9.9.3</rocksdb.version>
     <shrinkwrap.version>3.3.0</shrinkwrap.version>
     <slf4j.version>2.0.12</slf4j.version>
     <snakeyaml.version>2.0</snakeyaml.version>

tests/README.md

@@ -4,7 +4,8 @@
 
 ### Workaround for running the tests with Mac Apple Silicon
 
-The current version of the outdated maven plugin requires a workaround. 
+The current version of the outdated Maven plugin requires a workaround. This is also necessary for running the tests
+with an outdated Arquillian Cube version.
 
 Install socat
 ```bash
@@ -39,20 +40,43 @@ You can add the function to your shell profile (`~/.bashrc`, `~/.zshrc`, etc.).
 
 ### Support for connecting to docker network from Mac host
 
-You will also need to install [docker-mac-net-connect](https://github.com/chipmk/docker-mac-net-connect). It allows the tests to connect to the docker network from the Mac host.
+You will also need to install [docker-mac-net-connect](https://github.com/chipmk/docker-mac-net-connect). It allows the
+tests to connect to the Docker network from the Mac host.
+Disable host network mode in Docker Desktop settings before enabling `docker-mac-net-connect`, as they cannot be enabled
+simultaneously.
 
 ```bash
 brew install chipmk/tap/docker-mac-net-connect
 sudo brew services start chipmk/tap/docker-mac-net-connect
 ```
 
+You will also need to add the following to the Docker Desktop Docker Engine configuration on MacOS:
+```yaml
+{
+  "default-network-opts": {
+    "bridge": {
+      "com.docker.network.bridge.gateway_mode_ipv4": "nat-unprotected"
+    }
+  }
+}
+```
+
+Stop `docker-mac-net-connect` when you no longer need it:
+
+```bash
+sudo brew services stop chipmk/tap/docker-mac-net-connect
+```
+
 ## Running the tests
 
 Remember to start the unix socket proxy for docker as described in the previous section and set the `DOCKER_HOST` environment variable in the terminal where you run the tests.
 
 ### Building the docker images together with the project source code
 
 ```bash
+# remove possible remaining python client versions from previous builds
+git clean -fdx -- stream/clients/python
+# build the project and docker images
 mvn -B -nsu clean install -Pdocker -DskipTests
 docker images | grep apachebookkeeper
 ```

tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDirect.groovy

@@ -17,6 +17,8 @@
 */
 package org.apache.bookkeeper.tests.backwardcompat
 
+import static org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils.VERSION_4_1_x
+
 import com.github.dockerjava.api.DockerClient
 import org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils
 import org.apache.bookkeeper.tests.integration.utils.MavenClassLoader
@@ -49,8 +51,8 @@ class TestCompatUpgradeDirect {
         String currentVersion = BookKeeperClusterUtils.CURRENT_VERSION
         int numEntries = 10
 
-        Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, "4.1.0"))
-        def v410CL = MavenClassLoader.forBookKeeperVersion("4.1.0")
+        Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, VERSION_4_1_x))
+        def v410CL = MavenClassLoader.forBookKeeperVersion(VERSION_4_1_x)
         def v410BK = v410CL.newBookKeeper(zookeeper)
         def currentCL = MavenClassLoader.forBookKeeperVersion(currentVersion)
         def currentBK = currentCL.newBookKeeper(zookeeper)
@@ -102,7 +104,7 @@ class TestCompatUpgradeDirect {
 
         def currentCL = MavenClassLoader.forBookKeeperVersion(currentVersion)
         def currentBK = currentCL.newBookKeeper(zookeeper)
-        def v410CL = MavenClassLoader.forBookKeeperVersion("4.1.0")
+        def v410CL = MavenClassLoader.forBookKeeperVersion(VERSION_4_1_x)
         def v410BK = v410CL.newBookKeeper(zookeeper)
 
         try {

tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDowngrade.groovy

@@ -17,15 +17,14 @@
 */
 package org.apache.bookkeeper.tests.backwardcompat
 
+import static org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils.*
+
 import com.github.dockerjava.api.DockerClient
-import java.util.concurrent.TimeUnit
-import org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils
+import com.google.common.collect.Lists
 import org.apache.bookkeeper.tests.integration.utils.MavenClassLoader
 import org.jboss.arquillian.junit.Arquillian
 import org.jboss.arquillian.test.api.ArquillianResource
-import org.junit.AfterClass
 import org.junit.Assert
-import org.junit.BeforeClass
 import org.junit.FixMethodOrder
 import org.junit.Test
 import org.junit.runner.RunWith
@@ -57,62 +56,80 @@ class TestCompatUpgradeDowngrade {
     @Test
     public void atest_000_setUp() throws Exception {
         LOG.info("Running metaformat")
-        BookKeeperClusterUtils.legacyMetadataFormat(docker)
+        legacyMetadataFormat(docker)
 
         LOG.info("Setting ledger storage")
 
-        for (String version: BookKeeperClusterUtils.OLD_CLIENT_VERSIONS) {
-            BookKeeperClusterUtils.appendToAllBookieConf(docker, version,
+        List<String> versions = Lists.newArrayList(UPGRADE_DOWNGRADE_TEST_VERSIONS)
+        versions.add(CURRENT_VERSION)
+
+        File testRocksDbConfDir = new File(getClass().getClassLoader()
+                .getResource("TestCompatUpgradeDowngrade/conf/default_rocksdb.conf").toURI()).getParentFile()
+
+        boolean useRocksDbVersion5 = false
+        boolean useKxxHash = false
+        for (String version: versions) {
+            appendToAllBookieConf(docker, version,
                     "ledgerStorageClass",
                     "org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage")
+
+            try {
+                // format_version 5 has been supported since RocksDB 6.6.x
+                appendToAllBookieConf(docker, version,
+                        "dbStorage_rocksDB_format_version", "5")
+                // kxxHash has been supported for a very long time
+                appendToAllBookieConf(docker, version,
+                        "dbStorage_rocksDB_checksum_type", "kxxHash")
+
+                // copy rocksdb.conf to all bookies since some released versions don't contain the format_version
+                // which is necessary for backwards compatibility
+                copyToAllBookies(docker, version, testRocksDbConfDir)
+                appendToAllBookieConf(docker, version,
+                        "entryLocationRocksdbConf",
+                        "conf/entry_location_rocksdb.conf")
+                appendToAllBookieConf(docker, version,
+                        "ledgerMetadataRocksdbConf",
+                        "conf/ledger_metadata_rocksdb.conf")
+                appendToAllBookieConf(docker, version,
+                        "defaultRocksdbConf",
+                        "conf/default_rocksdb.conf")
+            } catch (Exception e) {
+                LOG.warn(version + ": Failed to set rocksdb configs, might be ok for some older version", e)
+            }
         }
-        BookKeeperClusterUtils.appendToAllBookieConf(docker, BookKeeperClusterUtils.CURRENT_VERSION,
-                "ledgerStorageClass",
-                "org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage")
     }
 
-
     // will ignore older non-supported versions
 
     @Test
-    public void upgradeDowngrade_010() {
-        upgradeDowngrade("4.12.1", "4.13.0")
-    }
-
-    @Test
-    public void upgradeDowngrade_011() {
-        upgradeDowngrade("4.13.0", "4.14.8")
-    }
-
-    @Test
-    public void upgradeDowngrade_012() {
-        upgradeDowngrade("4.14.8", "4.15.5")
+    public void upgradeDowngrade_for_4_14_x_and_4_15_x() {
+        upgradeDowngrade(VERSION_4_14_x, VERSION_4_15_x)
     }
 
     @Test
-    public void upgradeDowngrade_013() {
-        upgradeDowngrade("4.15.5", "4.16.5")
+    public void upgradeDowngrade_for_4_15_x_and_4_16_x() {
+        upgradeDowngrade(VERSION_4_15_x, VERSION_4_16_x)
     }
 
     @Test
-    public void upgradeDowngrade_014() {
-        upgradeDowngrade("4.16.5", "4.17.0")
+    public void upgradeDowngrade_for_4_16_x_and_4_17_x() {
+        upgradeDowngrade(VERSION_4_16_x, VERSION_4_17_x)
     }
 
     @Test
-    public void upgradeDowngrade_015() {
-        String currentVersion = BookKeeperClusterUtils.CURRENT_VERSION
-        upgradeDowngrade("4.17.0", currentVersion)
+    public void upgradeDowngrade_for_4_17_x_and_CurrentMaster() {
+        String currentVersion = CURRENT_VERSION
+        upgradeDowngrade(VERSION_4_17_x, currentVersion)
     }
 
     private void upgradeDowngrade(String oldVersion, String newVersion) throws Exception {
         LOG.info("Testing upgrade/downgrade to/from from {} to {}", oldVersion, newVersion)
 
-        String zookeeper = BookKeeperClusterUtils.zookeeperConnectString(docker)
+        String zookeeper = zookeeperConnectString(docker)
         int numEntries = 10
 
         LOG.info("Starting bookies with old version {}", oldVersion)
-        Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, oldVersion))
+        Assert.assertTrue(startAllBookiesWithVersion(docker, oldVersion))
 
         def oldCL = MavenClassLoader.forBookKeeperVersion(oldVersion)
         def oldBK = oldCL.newBookKeeper(zookeeper)
@@ -128,8 +145,8 @@ class TestCompatUpgradeDowngrade {
             testRead(ledger1, numEntries, oldBK, oldCL)
 
             LOG.info("Upgrade: Stopping all bookies, starting with new version {}", newVersion)
-            Assert.assertTrue(BookKeeperClusterUtils.stopAllBookies(docker))
-            Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, newVersion))
+            Assert.assertTrue(stopAllBookies(docker))
+            Assert.assertTrue(startAllBookiesWithVersion(docker, newVersion))
 
             LOG.info("Reading ledger with old client")
             testRead(ledger0, numEntries, oldBK, oldCL)
@@ -140,8 +157,8 @@ class TestCompatUpgradeDowngrade {
             testRead(ledger1, numEntries, newBK, newCL)
 
             LOG.info("Downgrade: Stopping all bookies, starting with old version {}", oldVersion)
-            Assert.assertTrue(BookKeeperClusterUtils.stopAllBookies(docker))
-            Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, oldVersion))
+            Assert.assertTrue(stopAllBookies(docker))
+            Assert.assertTrue(startAllBookiesWithVersion(docker, oldVersion))
 
             LOG.info("Reading ledgers with old client")
             testRead(ledger0, numEntries, oldBK, oldCL)
@@ -160,7 +177,7 @@ class TestCompatUpgradeDowngrade {
             testRead(ledger4, numEntries, newBK, newCL)
 
         } finally {
-            BookKeeperClusterUtils.stopAllBookies(docker)
+            stopAllBookies(docker)
 
             newBK.close()
             newCL.close()

tests/backward-compat/upgrade-direct/src/test/resources/TestCompatUpgradeDowngrade/conf/default_rocksdb.conf

@@ -0,0 +1,40 @@
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# When modifying this file, please also modify the configuration files(at
+# bookkeeper-server/src/test/resources/conf) in the
+# test case to ensure unit test coverage.
+
+[DBOptions]
+ # set by jni: options.setCreateIfMissing
+ create_if_missing=true
+ # set by jni: options.setInfoLogLevel
+ info_log_level=INFO_LEVEL
+ # set by jni: options.setKeepLogFileNum
+ keep_log_file_num=30
+ # set by jni: options.setLogFileTimeToRoll
+ log_file_time_to_roll=86400
+
+ [CFOptions "default"]
+ #no default setting in CFOptions
+
+[TableOptions/BlockBasedTable "default"]
+ # set by jni: tableOptions.setFormatVersion
+ format_version=5
+ # set by jni: tableOptions.setChecksumType
+ checksum=kxxHash