diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index a3d6f85db47..9ecb02fc019 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -271,7 +271,7 @@ Apache Software License, Version 2. - lib/org.eclipse.jetty-jetty-servlet-9.4.55.v20240627.jar [22] - lib/org.eclipse.jetty-jetty-util-9.4.55.v20240627.jar [22] - lib/org.eclipse.jetty-jetty-util-ajax-9.4.55.v20240627.jar [22] -- lib/org.rocksdb-rocksdbjni-7.10.2.jar [23] +- lib/org.rocksdb-rocksdbjni-9.9.3.jar [23] - lib/com.beust-jcommander-1.82.jar [24] - lib/com.yahoo.datasketches-memory-0.8.3.jar [25] - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25] @@ -378,7 +378,7 @@ Apache Software License, Version 2. [20] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6 [21] Source available at https://github.com/apache/zookeeper/tree/release-3.8.0 [22] Source available at https://github.com/eclipse/jetty.project/tree/jetty-9.4.48.v20220622 -[23] Source available at https://github.com/facebook/rocksdb/tree/v7.10.2 +[23] Source available at https://github.com/facebook/rocksdb/tree/v9.9.3 [24] Source available at https://github.com/cbeust/jcommander/tree/1.82 [25] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3 [26] Source available at https://github.com/lz4/lz4-java/tree/1.3.0 @@ -639,7 +639,7 @@ This private header is also used by Apple's open source * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h ------------------------------------------------------------------------------------ -lib/org.rocksdb-rocksdbjni-7.10.2.jar is derived from leveldb, which is under the following license. +lib/org.rocksdb-rocksdbjni-9.9.3.jar is derived from leveldb, which is under the following license. Copyright (c) 2011 The LevelDB Authors. All rights reserved. diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index e473a34f477..fa5fd51d576 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -271,7 +271,7 @@ Apache Software License, Version 2. - lib/org.eclipse.jetty-jetty-servlet-9.4.55.v20240627.jar [22] - lib/org.eclipse.jetty-jetty-util-9.4.55.v20240627.jar [22] - lib/org.eclipse.jetty-jetty-util-ajax-9.4.55.v20240627.jar [22] -- lib/org.rocksdb-rocksdbjni-7.10.2.jar [23] +- lib/org.rocksdb-rocksdbjni-9.9.3.jar [23] - lib/com.beust-jcommander-1.82.jar [24] - lib/com.yahoo.datasketches-memory-0.8.3.jar [25] - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25] @@ -374,7 +374,7 @@ Apache Software License, Version 2. [20] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6 [21] Source available at https://github.com/apache/zookeeper/tree/release-3.8.0 [22] Source available at https://github.com/eclipse/jetty.project/tree/jetty-9.4.48.v20220622 -[23] Source available at https://github.com/facebook/rocksdb/tree/v7.10.2 +[23] Source available at https://github.com/facebook/rocksdb/tree/v9.9.3 [24] Source available at https://github.com/cbeust/jcommander/tree/1.82 [25] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3 [26] Source available at https://github.com/lz4/lz4-java/tree/1.3.0 @@ -634,7 +634,7 @@ This private header is also used by Apple's open source * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h ------------------------------------------------------------------------------------ -lib/org.rocksdb-rocksdbjni-7.10.2.jar is derived from leveldb, which is under the following license. +lib/org.rocksdb-rocksdbjni-9.9.3.jar is derived from leveldb, which is under the following license. Copyright (c) 2011 The LevelDB Authors. All rights reserved. diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java index c9e72847fbc..12acbd90f11 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java @@ -48,6 +48,7 @@ import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.CompressionType; +import org.rocksdb.ConfigOptions; import org.rocksdb.DBOptions; import org.rocksdb.Env; import org.rocksdb.InfoLogLevel; @@ -155,8 +156,10 @@ private RocksDB initializeRocksDBWithConfFile(String basePath, String subPath, D DBOptions dbOptions = new DBOptions(); final List cfDescs = new ArrayList<>(); final List cfHandles = new ArrayList<>(); - try { - OptionsUtil.loadOptionsFromFile(dbFilePath, Env.getDefault(), dbOptions, cfDescs, false); + try (final ConfigOptions cfgOpts = new ConfigOptions() + .setIgnoreUnknownOptions(false) + .setEnv(Env.getDefault())) { + OptionsUtil.loadOptionsFromFile(cfgOpts, dbFilePath, dbOptions, cfDescs); // Configure file path String logPath = conf.getString(ROCKSDB_LOG_PATH, ""); if (!logPath.isEmpty()) { diff --git a/pom.xml b/pom.xml index de32adf8cb0..bc2b53a76d3 100644 --- a/pom.xml +++ b/pom.xml @@ -167,7 +167,7 @@ ${protobuf.version} ${grpc.version} 0.9.11 - 7.10.2 + 9.9.3 3.3.0 2.0.12 2.0 diff --git a/tests/README.md b/tests/README.md index dff705c4bca..53f5410464d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -4,7 +4,8 @@ ### Workaround for running the tests with Mac Apple Silicon -The current version of the outdated maven plugin requires a workaround. +The current version of the outdated Maven plugin requires a workaround. This is also necessary for running the tests +with an outdated Arquillian Cube version. Install socat ```bash @@ -39,13 +40,33 @@ You can add the function to your shell profile (`~/.bashrc`, `~/.zshrc`, etc.). ### Support for connecting to docker network from Mac host -You will also need to install [docker-mac-net-connect](https://github.com/chipmk/docker-mac-net-connect). It allows the tests to connect to the docker network from the Mac host. +You will also need to install [docker-mac-net-connect](https://github.com/chipmk/docker-mac-net-connect). It allows the +tests to connect to the Docker network from the Mac host. +Disable host network mode in Docker Desktop settings before enabling `docker-mac-net-connect`, as they cannot be enabled +simultaneously. ```bash brew install chipmk/tap/docker-mac-net-connect sudo brew services start chipmk/tap/docker-mac-net-connect ``` +You will also need to add the following to the Docker Desktop Docker Engine configuration on MacOS: +```yaml +{ + "default-network-opts": { + "bridge": { + "com.docker.network.bridge.gateway_mode_ipv4": "nat-unprotected" + } + } +} +``` + +Stop `docker-mac-net-connect` when you no longer need it: + +```bash +sudo brew services stop chipmk/tap/docker-mac-net-connect +``` + ## Running the tests Remember to start the unix socket proxy for docker as described in the previous section and set the `DOCKER_HOST` environment variable in the terminal where you run the tests. @@ -53,6 +74,9 @@ Remember to start the unix socket proxy for docker as described in the previous ### Building the docker images together with the project source code ```bash +# remove possible remaining python client versions from previous builds +git clean -fdx -- stream/clients/python +# build the project and docker images mvn -B -nsu clean install -Pdocker -DskipTests docker images | grep apachebookkeeper ``` diff --git a/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDirect.groovy b/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDirect.groovy index cb71dc27c42..c66393be633 100644 --- a/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDirect.groovy +++ b/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDirect.groovy @@ -17,6 +17,8 @@ */ package org.apache.bookkeeper.tests.backwardcompat +import static org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils.VERSION_4_1_x + import com.github.dockerjava.api.DockerClient import org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils import org.apache.bookkeeper.tests.integration.utils.MavenClassLoader @@ -49,8 +51,8 @@ class TestCompatUpgradeDirect { String currentVersion = BookKeeperClusterUtils.CURRENT_VERSION int numEntries = 10 - Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, "4.1.0")) - def v410CL = MavenClassLoader.forBookKeeperVersion("4.1.0") + Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, VERSION_4_1_x)) + def v410CL = MavenClassLoader.forBookKeeperVersion(VERSION_4_1_x) def v410BK = v410CL.newBookKeeper(zookeeper) def currentCL = MavenClassLoader.forBookKeeperVersion(currentVersion) def currentBK = currentCL.newBookKeeper(zookeeper) @@ -102,7 +104,7 @@ class TestCompatUpgradeDirect { def currentCL = MavenClassLoader.forBookKeeperVersion(currentVersion) def currentBK = currentCL.newBookKeeper(zookeeper) - def v410CL = MavenClassLoader.forBookKeeperVersion("4.1.0") + def v410CL = MavenClassLoader.forBookKeeperVersion(VERSION_4_1_x) def v410BK = v410CL.newBookKeeper(zookeeper) try { diff --git a/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDowngrade.groovy b/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDowngrade.groovy index 74e0c9486a4..e448cc27b41 100644 --- a/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDowngrade.groovy +++ b/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDowngrade.groovy @@ -17,15 +17,14 @@ */ package org.apache.bookkeeper.tests.backwardcompat +import static org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils.* + import com.github.dockerjava.api.DockerClient -import java.util.concurrent.TimeUnit -import org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils +import com.google.common.collect.Lists import org.apache.bookkeeper.tests.integration.utils.MavenClassLoader import org.jboss.arquillian.junit.Arquillian import org.jboss.arquillian.test.api.ArquillianResource -import org.junit.AfterClass import org.junit.Assert -import org.junit.BeforeClass import org.junit.FixMethodOrder import org.junit.Test import org.junit.runner.RunWith @@ -57,62 +56,80 @@ class TestCompatUpgradeDowngrade { @Test public void atest_000_setUp() throws Exception { LOG.info("Running metaformat") - BookKeeperClusterUtils.legacyMetadataFormat(docker) + legacyMetadataFormat(docker) LOG.info("Setting ledger storage") - for (String version: BookKeeperClusterUtils.OLD_CLIENT_VERSIONS) { - BookKeeperClusterUtils.appendToAllBookieConf(docker, version, + List versions = Lists.newArrayList(UPGRADE_DOWNGRADE_TEST_VERSIONS) + versions.add(CURRENT_VERSION) + + File testRocksDbConfDir = new File(getClass().getClassLoader() + .getResource("TestCompatUpgradeDowngrade/conf/default_rocksdb.conf").toURI()).getParentFile() + + boolean useRocksDbVersion5 = false + boolean useKxxHash = false + for (String version: versions) { + appendToAllBookieConf(docker, version, "ledgerStorageClass", "org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage") + + try { + // format_version 5 has been supported since RocksDB 6.6.x + appendToAllBookieConf(docker, version, + "dbStorage_rocksDB_format_version", "5") + // kxxHash has been supported for a very long time + appendToAllBookieConf(docker, version, + "dbStorage_rocksDB_checksum_type", "kxxHash") + + // copy rocksdb.conf to all bookies since some released versions don't contain the format_version + // which is necessary for backwards compatibility + copyToAllBookies(docker, version, testRocksDbConfDir) + appendToAllBookieConf(docker, version, + "entryLocationRocksdbConf", + "conf/entry_location_rocksdb.conf") + appendToAllBookieConf(docker, version, + "ledgerMetadataRocksdbConf", + "conf/ledger_metadata_rocksdb.conf") + appendToAllBookieConf(docker, version, + "defaultRocksdbConf", + "conf/default_rocksdb.conf") + } catch (Exception e) { + LOG.warn(version + ": Failed to set rocksdb configs, might be ok for some older version", e) + } } - BookKeeperClusterUtils.appendToAllBookieConf(docker, BookKeeperClusterUtils.CURRENT_VERSION, - "ledgerStorageClass", - "org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage") } - // will ignore older non-supported versions @Test - public void upgradeDowngrade_010() { - upgradeDowngrade("4.12.1", "4.13.0") - } - - @Test - public void upgradeDowngrade_011() { - upgradeDowngrade("4.13.0", "4.14.8") - } - - @Test - public void upgradeDowngrade_012() { - upgradeDowngrade("4.14.8", "4.15.5") + public void upgradeDowngrade_for_4_14_x_and_4_15_x() { + upgradeDowngrade(VERSION_4_14_x, VERSION_4_15_x) } @Test - public void upgradeDowngrade_013() { - upgradeDowngrade("4.15.5", "4.16.5") + public void upgradeDowngrade_for_4_15_x_and_4_16_x() { + upgradeDowngrade(VERSION_4_15_x, VERSION_4_16_x) } @Test - public void upgradeDowngrade_014() { - upgradeDowngrade("4.16.5", "4.17.0") + public void upgradeDowngrade_for_4_16_x_and_4_17_x() { + upgradeDowngrade(VERSION_4_16_x, VERSION_4_17_x) } @Test - public void upgradeDowngrade_015() { - String currentVersion = BookKeeperClusterUtils.CURRENT_VERSION - upgradeDowngrade("4.17.0", currentVersion) + public void upgradeDowngrade_for_4_17_x_and_CurrentMaster() { + String currentVersion = CURRENT_VERSION + upgradeDowngrade(VERSION_4_17_x, currentVersion) } private void upgradeDowngrade(String oldVersion, String newVersion) throws Exception { LOG.info("Testing upgrade/downgrade to/from from {} to {}", oldVersion, newVersion) - String zookeeper = BookKeeperClusterUtils.zookeeperConnectString(docker) + String zookeeper = zookeeperConnectString(docker) int numEntries = 10 LOG.info("Starting bookies with old version {}", oldVersion) - Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, oldVersion)) + Assert.assertTrue(startAllBookiesWithVersion(docker, oldVersion)) def oldCL = MavenClassLoader.forBookKeeperVersion(oldVersion) def oldBK = oldCL.newBookKeeper(zookeeper) @@ -128,8 +145,8 @@ class TestCompatUpgradeDowngrade { testRead(ledger1, numEntries, oldBK, oldCL) LOG.info("Upgrade: Stopping all bookies, starting with new version {}", newVersion) - Assert.assertTrue(BookKeeperClusterUtils.stopAllBookies(docker)) - Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, newVersion)) + Assert.assertTrue(stopAllBookies(docker)) + Assert.assertTrue(startAllBookiesWithVersion(docker, newVersion)) LOG.info("Reading ledger with old client") testRead(ledger0, numEntries, oldBK, oldCL) @@ -140,8 +157,8 @@ class TestCompatUpgradeDowngrade { testRead(ledger1, numEntries, newBK, newCL) LOG.info("Downgrade: Stopping all bookies, starting with old version {}", oldVersion) - Assert.assertTrue(BookKeeperClusterUtils.stopAllBookies(docker)) - Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, oldVersion)) + Assert.assertTrue(stopAllBookies(docker)) + Assert.assertTrue(startAllBookiesWithVersion(docker, oldVersion)) LOG.info("Reading ledgers with old client") testRead(ledger0, numEntries, oldBK, oldCL) @@ -160,7 +177,7 @@ class TestCompatUpgradeDowngrade { testRead(ledger4, numEntries, newBK, newCL) } finally { - BookKeeperClusterUtils.stopAllBookies(docker) + stopAllBookies(docker) newBK.close() newCL.close() diff --git a/tests/backward-compat/upgrade-direct/src/test/resources/TestCompatUpgradeDowngrade/conf/default_rocksdb.conf b/tests/backward-compat/upgrade-direct/src/test/resources/TestCompatUpgradeDowngrade/conf/default_rocksdb.conf new file mode 100644 index 00000000000..ccedbb79ebb --- /dev/null +++ b/tests/backward-compat/upgrade-direct/src/test/resources/TestCompatUpgradeDowngrade/conf/default_rocksdb.conf @@ -0,0 +1,40 @@ +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# When modifying this file, please also modify the configuration files(at +# bookkeeper-server/src/test/resources/conf) in the +# test case to ensure unit test coverage. + +[DBOptions] + # set by jni: options.setCreateIfMissing + create_if_missing=true + # set by jni: options.setInfoLogLevel + info_log_level=INFO_LEVEL + # set by jni: options.setKeepLogFileNum + keep_log_file_num=30 + # set by jni: options.setLogFileTimeToRoll + log_file_time_to_roll=86400 + + [CFOptions "default"] + #no default setting in CFOptions + +[TableOptions/BlockBasedTable "default"] + # set by jni: tableOptions.setFormatVersion + format_version=5 + # set by jni: tableOptions.setChecksumType + checksum=kxxHash \ No newline at end of file diff --git a/tests/backward-compat/upgrade-direct/src/test/resources/TestCompatUpgradeDowngrade/conf/entry_location_rocksdb.conf b/tests/backward-compat/upgrade-direct/src/test/resources/TestCompatUpgradeDowngrade/conf/entry_location_rocksdb.conf new file mode 100644 index 00000000000..e4dc394243f --- /dev/null +++ b/tests/backward-compat/upgrade-direct/src/test/resources/TestCompatUpgradeDowngrade/conf/entry_location_rocksdb.conf @@ -0,0 +1,73 @@ +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# When modifying this file, please also modify the configuration files(at +# bookkeeper-server/src/test/resources/conf) in the +# test case to ensure unit test coverage. + +[DBOptions] + # set by jni: options.setCreateIfMissing + create_if_missing=true + # set by jni: options.setInfoLogLevel + info_log_level=INFO_LEVEL + # set by jni: options.setKeepLogFileNum + keep_log_file_num=30 + # set by jni: options.setLogFileTimeToRoll + log_file_time_to_roll=86400 + # set by jni: options.setMaxBackgroundJobs or options.setIncreaseParallelism + max_background_jobs=32 + # set by jni: options.setMaxSubcompactions + max_subcompactions=1 + # set by jni: options.setMaxTotalWalSize + max_total_wal_size=536870912 + # set by jni: options.setMaxOpenFiles + max_open_files=-1 + # set by jni: options.setDeleteObsoleteFilesPeriodMicros + delete_obsolete_files_period_micros=3600000000 + +[CFOptions "default"] + # set by jni: options.setCompressionType + compression=kLZ4Compression + # set by jni: options.setWriteBufferSize + write_buffer_size=67108864 + # set by jni: options.setMaxWriteBufferNumber + max_write_buffer_number=4 + # set by jni: options.setNumLevels + num_levels=7 + # set by jni: options.setLevelZeroFileNumCompactionTrigger + level0_file_num_compaction_trigger=4 + # set by jni: options.setMaxBytesForLevelBase + max_bytes_for_level_base=268435456 + # set by jni: options.setTargetFileSizeBase + target_file_size_base=67108864 + # set by jni: options.setLevelCompactionDynamicLevelBytes + level_compaction_dynamic_level_bytes=true + +[TableOptions/BlockBasedTable "default"] + # set by jni: tableOptions.setBlockSize + block_size=65536 + # set by jni: tableOptions.setBlockCache, default value is: maxDirectMemory() / ledgerDirsSize / 10; + block_cache=206150041 + # set by jni: tableOptions.setFormatVersion + format_version=5 + # set by jni: tableOptions.setChecksumType + checksum=kxxHash + # set by jni: tableOptions.setFilterPolicy, bloomfilter:[bits_per_key]:[use_block_based_builder] + filter_policy=rocksdb.BloomFilter:10:false + # set by jni: tableOptions.setCacheIndexAndFilterBlocks + cache_index_and_filter_blocks=true \ No newline at end of file diff --git a/tests/backward-compat/upgrade-direct/src/test/resources/TestCompatUpgradeDowngrade/conf/ledger_metadata_rocksdb.conf b/tests/backward-compat/upgrade-direct/src/test/resources/TestCompatUpgradeDowngrade/conf/ledger_metadata_rocksdb.conf new file mode 100644 index 00000000000..ccedbb79ebb --- /dev/null +++ b/tests/backward-compat/upgrade-direct/src/test/resources/TestCompatUpgradeDowngrade/conf/ledger_metadata_rocksdb.conf @@ -0,0 +1,40 @@ +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# When modifying this file, please also modify the configuration files(at +# bookkeeper-server/src/test/resources/conf) in the +# test case to ensure unit test coverage. + +[DBOptions] + # set by jni: options.setCreateIfMissing + create_if_missing=true + # set by jni: options.setInfoLogLevel + info_log_level=INFO_LEVEL + # set by jni: options.setKeepLogFileNum + keep_log_file_num=30 + # set by jni: options.setLogFileTimeToRoll + log_file_time_to_roll=86400 + + [CFOptions "default"] + #no default setting in CFOptions + +[TableOptions/BlockBasedTable "default"] + # set by jni: tableOptions.setFormatVersion + format_version=5 + # set by jni: tableOptions.setChecksumType + checksum=kxxHash \ No newline at end of file diff --git a/tests/backward-compat/upgrade/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgrade.groovy b/tests/backward-compat/upgrade/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgrade.groovy index c987bd5770d..20475f14846 100644 --- a/tests/backward-compat/upgrade/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgrade.groovy +++ b/tests/backward-compat/upgrade/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgrade.groovy @@ -17,6 +17,8 @@ */ package org.apache.bookkeeper.tests.backwardcompat +import static org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils.* + import com.github.dockerjava.api.DockerClient import org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils import org.apache.bookkeeper.tests.integration.utils.MavenClassLoader @@ -122,84 +124,83 @@ class TestCompatUpgrade { @Test public void test_000() throws Exception { BookKeeperClusterUtils.legacyMetadataFormat(docker) - Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, "4.8.2")) + Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, VERSION_4_8_x)) } @Test - public void test_001_482to492() throws Exception { - testUpgrade("4.8.2", "4.9.2") + public void test_001_48xto49x() throws Exception { + testUpgrade(VERSION_4_8_x, VERSION_4_9_x) } @Test - public void test_002_492to4100() throws Exception { - testUpgrade("4.9.2", "4.10.0") + public void test_002_49xto410x() throws Exception { + testUpgrade(VERSION_4_9_x, VERSION_4_10_x) } @Test - public void test_003_4100to4111() throws Exception { - testUpgrade("4.10.0", "4.11.1") + public void test_003_410xto411x() throws Exception { + testUpgrade(VERSION_4_10_x, VERSION_4_11_x) } @Test - public void test_004_4111to4121() throws Exception { - testUpgrade("4.11.1", "4.12.1") + public void test_004_411xto412x() throws Exception { + testUpgrade(VERSION_4_11_x, VERSION_4_12_x) } @Test - public void test_005_4121to4130() throws Exception { - testUpgrade("4.12.1", "4.13.0") + public void test_005_412xto413x() throws Exception { + testUpgrade(VERSION_4_12_x, VERSION_4_13_x) } @Test - public void test_006_4130to4148() throws Exception { - testUpgrade("4.13.0", "4.14.8") + public void test_006_413xto414x() throws Exception { + testUpgrade(VERSION_4_13_x, VERSION_4_14_x) } @Test - public void test_007_4148to4155() throws Exception { - testUpgrade("4.14.8", "4.15.5") + public void test_007_414xto415x() throws Exception { + testUpgrade(VERSION_4_14_x, VERSION_4_15_x) } @Test - public void test_007_4148to4155_crc32c() throws Exception { - testUpgrade("4.14.8", "4.15.5", "CRC32C") + public void test_007_414xto415x_crc32c() throws Exception { + testUpgrade(VERSION_4_14_x, VERSION_4_15_x, "CRC32C") } @Test - public void test_008_4155to4165() throws Exception { - testUpgrade("4.15.5", "4.16.5") + public void test_008_415xto416x() throws Exception { + testUpgrade(VERSION_4_15_x, VERSION_4_16_x) } @Test - public void test_008_4155to4165_crc32c() throws Exception { - testUpgrade("4.15.5", "4.16.5", "CRC32C") + public void test_008_415xto416x_crc32c() throws Exception { + testUpgrade(VERSION_4_15_x, VERSION_4_16_x, "CRC32C") } @Test - public void test_008_4165to4170_crc32c() throws Exception { - testUpgrade("4.16.5", "4.17.0", "CRC32C") + public void test_008_416xto417x_crc32c() throws Exception { + testUpgrade(VERSION_4_16_x, VERSION_4_17_x, "CRC32C") } @Test - public void test_009_4165toCurrentMaster() throws Exception { - testUpgrade("4.17.0", BookKeeperClusterUtils.CURRENT_VERSION) + public void test_009_417xtoCurrentMaster() throws Exception { + testUpgrade(VERSION_4_17_x, CURRENT_VERSION) } @Test - public void test_009_4165toCurrentMaster_crc32c() throws Exception { - testUpgrade("4.17.0", BookKeeperClusterUtils.CURRENT_VERSION, "CRC32C") + public void test_009_417xtoCurrentMaster_crc32c() throws Exception { + testUpgrade(VERSION_4_17_x, CURRENT_VERSION, "CRC32C") } // old version pulsar upgrade tests @Test - public void test_010_4100to4148_crc32c() throws Exception { - testUpgrade("4.10.0", "4.14.8", "CRC32C") + public void test_010_410xto414x_crc32c() throws Exception { + testUpgrade(VERSION_4_10_x, VERSION_4_14_x, "CRC32C") } // old version pulsar upgrade tests @Test - public void test_010_4100to4170_crc32c() throws Exception { - testUpgrade("4.10.0", "4.17.0", "CRC32C") + public void test_010_410xto417x_crc32c() throws Exception { + testUpgrade(VERSION_4_10_x, VERSION_4_17_x, "CRC32C") } - } diff --git a/tests/docker-images/all-released-versions-image/Dockerfile b/tests/docker-images/all-released-versions-image/Dockerfile index 6bbdad2c309..0051d1b7e61 100644 --- a/tests/docker-images/all-released-versions-image/Dockerfile +++ b/tests/docker-images/all-released-versions-image/Dockerfile @@ -51,8 +51,8 @@ RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.12.1/bookke RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.13.0/bookkeeper-server-4.13.0-bin.tar.gz{,.sha512,.asc} RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.14.8/bookkeeper-server-4.14.8-bin.tar.gz{,.sha512,.asc} RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.15.5/bookkeeper-server-4.15.5-bin.tar.gz{,.sha512,.asc} -RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.16.5/bookkeeper-server-4.16.5-bin.tar.gz{,.sha512,.asc} -RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.17.0/bookkeeper-server-4.17.0-bin.tar.gz{,.sha512,.asc} +RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.16.6/bookkeeper-server-4.16.6-bin.tar.gz{,.sha512,.asc} +RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.17.1/bookkeeper-server-4.17.1-bin.tar.gz{,.sha512,.asc} RUN wget -nv https://dist.apache.org/repos/dist/release/bookkeeper/KEYS RUN wget -nv http://svn.apache.org/repos/asf/zookeeper/bookkeeper/dist/KEYS?p=1620552 -O KEYS.old diff --git a/tests/integration-tests-topologies/src/main/resources/cube-definitions/3-node-all-version-unstarted.yaml b/tests/integration-tests-topologies/src/main/resources/cube-definitions/3-node-all-version-unstarted.yaml index a422f4372fc..5cd627f258c 100644 --- a/tests/integration-tests-topologies/src/main/resources/cube-definitions/3-node-all-version-unstarted.yaml +++ b/tests/integration-tests-topologies/src/main/resources/cube-definitions/3-node-all-version-unstarted.yaml @@ -22,7 +22,7 @@ networks: driver: bridge zookeeper*: - image: zookeeper:3.6.2 + image: zookeeper:3.9.3 await: strategy: org.apache.bookkeeper.tests.integration.utils.ZooKeeperAwaitStrategy aliases: diff --git a/tests/integration-tests-utils/pom.xml b/tests/integration-tests-utils/pom.xml index 0a972558b95..b5aead05006 100644 --- a/tests/integration-tests-utils/pom.xml +++ b/tests/integration-tests-utils/pom.xml @@ -74,6 +74,15 @@ javax.ws.rs-api + + org.apache.logging.log4j + log4j-core + + + org.apache.logging.log4j + log4j-slf4j2-impl + + diff --git a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java index a6c8e8604f7..b8eb44516e8 100644 --- a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java +++ b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java @@ -18,6 +18,8 @@ package org.apache.bookkeeper.tests.integration.utils; import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.command.CopyArchiveToContainerCmd; +import java.io.File; import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -39,12 +41,27 @@ */ public class BookKeeperClusterUtils { public static final String CURRENT_VERSION = System.getProperty("currentVersion"); + + public static final String VERSION_4_1_x = "4.1.0"; + public static final String VERSION_4_8_x = "4.8.2"; + public static final String VERSION_4_9_x = "4.9.2"; + public static final String VERSION_4_10_x = "4.10.0"; + public static final String VERSION_4_11_x = "4.11.1"; + public static final String VERSION_4_12_x = "4.12.1"; + public static final String VERSION_4_13_x = "4.13.0"; + public static final String VERSION_4_14_x = "4.14.8"; + public static final String VERSION_4_15_x = "4.15.5"; + public static final String VERSION_4_16_x = "4.16.6"; + public static final String VERSION_4_17_x = "4.17.1"; + public static final List OLD_CLIENT_VERSIONS = - Arrays.asList("4.8.2", "4.9.2", "4.10.0", "4.11.1", "4.12.1", - "4.13.0", "4.14.8", "4.15.5", "4.16.5", "4.17.0"); + Arrays.asList(VERSION_4_8_x, VERSION_4_9_x, VERSION_4_10_x, VERSION_4_11_x, VERSION_4_12_x, + VERSION_4_13_x, VERSION_4_14_x, VERSION_4_15_x, VERSION_4_16_x, VERSION_4_17_x); private static final List OLD_CLIENT_VERSIONS_WITH_CURRENT_LEDGER_METADATA_FORMAT = - Arrays.asList("4.9.2", "4.10.0", "4.11.1", "4.12.1", - "4.13.0", "4.14.8", "4.15.5", "4.16.5", "4.17.0"); + Arrays.asList(VERSION_4_9_x, VERSION_4_10_x, VERSION_4_11_x, VERSION_4_12_x, + VERSION_4_13_x, VERSION_4_14_x, VERSION_4_15_x, VERSION_4_16_x, VERSION_4_17_x); + public static final List UPGRADE_DOWNGRADE_TEST_VERSIONS = + Arrays.asList(VERSION_4_14_x, VERSION_4_15_x, VERSION_4_16_x, VERSION_4_17_x); private static final Logger LOG = LoggerFactory.getLogger(BookKeeperClusterUtils.class); @@ -143,16 +160,38 @@ public static void updateBookieConf(DockerClient docker, String containerId, DockerUtils.runCommand(docker, containerId, "sed", "-i", "-e", sedProgram, confFile); } - public static void appendToAllBookieConf(DockerClient docker, String version, String key, String value) + public static void copyToAllBookies(DockerClient docker, String version, File sourceDirectory) + throws Exception { + for (String containerId : allBookies()) { + CopyArchiveToContainerCmd copyArchiveToContainerCmd = docker.copyArchiveToContainerCmd(containerId); + copyArchiveToContainerCmd.withHostResource(sourceDirectory.getAbsolutePath()); + copyArchiveToContainerCmd.withRemotePath("/opt/bookkeeper/" + version); + copyArchiveToContainerCmd.exec(); + } + } + + public static void appendToAllBookieConf(DockerClient docker, String version, String confFile0, + String key, String value) throws Exception { for (String b : allBookies()) { - appendToBookieConf(docker, b, version, key, value); + appendToBookieConf(docker, b, version, confFile0, key, value); } } + public static void appendToAllBookieConf(DockerClient docker, String version, + String key, String value) + throws Exception { + appendToAllBookieConf(docker, version, "conf/bk_server.conf", key, value); + } + public static void appendToBookieConf(DockerClient docker, String containerId, String version, String key, String value) throws Exception { - String confFile = "/opt/bookkeeper/" + version + "/conf/bk_server.conf"; + appendToBookieConf(docker, containerId, version, "conf/bk_server.conf", key, value); + } + + public static void appendToBookieConf(DockerClient docker, String containerId, + String version, String confFile0, String key, String value) throws Exception { + String confFile = "/opt/bookkeeper/" + version + "/" + confFile0; String sedProgram = String.format("$a%s=%s", key, value); DockerUtils.runCommand(docker, containerId, "sed", "-i", "-e", sedProgram, confFile); }