Skip to content

Commit 2cebd62

Browse files
mrambachermrambacher
committed
Merge RocksDB 6.14.6 into RocksDB Cloud
1 parent cf7ed24 commit 2cebd62

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+1390
-2228
lines changed

‎Makefile‎

Lines changed: 13 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -961,12 +961,6 @@ else
961961
parallel_com = '{} >& t/log-{/}'
962962
endif
963963

964-
ifeq ($(PRINT_PARALLEL_OUTPUTS), 1)
965-
parallel_com = '{}'
966-
else
967-
parallel_com = '{} >& t/log-{/}'
968-
endif
969-
970964
.PHONY: check_0
971965
check_0:
972966
$(AM_V_GEN)export TEST_TMPDIR=$(TMPD); \
@@ -1357,7 +1351,7 @@ block_cache_trace_analyzer: $(OBJ_DIR)/tools/block_cache_analyzer/block_cache_tr
13571351
$(AM_LINK)
13581352

13591353
ifeq ($(USE_FOLLY_DISTRIBUTED_MUTEX),1)
1360-
folly_synchronization_distributed_mutex_test: $(OBJ_DIR)/third-party/folly/folly/synchronization/test/DistributedMutexTest.o $(TEST_LIBRARY) $(LIBRARY)
1354+
folly_synchronization_distributed_mutex ioooooook_test: $(OBJ_DIR)/third-party/folly/folly/synchronization/test/DistributedMutexTest.o $(TEST_LIBRARY) $(LIBRARY)
13611355
$(AM_LINK)
13621356
endif
13631357

@@ -1867,6 +1861,18 @@ ldb_cmd_test: $(OBJ_DIR)/tools/ldb_cmd_test.o $(TOOLS_LIBRARY) $(TEST_LIBRARY) $
18671861
ldb: $(OBJ_DIR)/tools/ldb.o $(TOOLS_LIBRARY) $(LIBRARY)
18681862
$(AM_LINK)
18691863

1864+
remote_compaction_test: cloud/remote_compaction_test.o $(TEST_LIBRARY) $(LIBRARY)
1865+
$(AM_LINK)
1866+
1867+
db_cloud_test: cloud/db_cloud_test.o $(TEST_LIBRARY) $(LIBRARY)
1868+
$(AM_LINK)
1869+
1870+
cloud_manifest_test: cloud/cloud_manifest_test.o $(TEST_LIBRARY) $(LIBRARY)
1871+
$(AM_LINK)
1872+
1873+
cloud_scheduler_test: cloud/cloud_scheduler_test.o $(TEST_LIBRARY) $(LIBRARY)
1874+
$(AM_LINK)
1875+
18701876
iostats_context_test: $(OBJ_DIR)/monitoring/iostats_context_test.o $(TEST_LIBRARY) $(LIBRARY)
18711877
$(AM_V_CCLD)$(CXX) $^ $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
18721878

@@ -1945,18 +1951,6 @@ io_tracer_parser_test: $(OBJ_DIR)/tools/io_tracer_parser_test.o $(OBJ_DIR)/tools
19451951
io_tracer_parser: $(OBJ_DIR)/tools/io_tracer_parser.o $(TOOLS_LIBRARY) $(LIBRARY)
19461952
$(AM_LINK)
19471953

1948-
blob_file_addition_test: db/blob/blob_file_addition_test.o $(LIBOBJECTS) $(TESTHARNESS)
1949-
$(AM_LINK)
1950-
1951-
blob_file_garbage_test: db/blob/blob_file_garbage_test.o $(LIBOBJECTS) $(TESTHARNESS)
1952-
$(AM_LINK)
1953-
1954-
timer_test: util/timer_test.o $(LIBOBJECTS) $(TESTHARNESS)
1955-
$(AM_LINK)
1956-
1957-
testutil_test: test_util/testutil_test.o $(LIBOBJECTS) $(TESTHARNESS)
1958-
$(AM_LINK)
1959-
19601954
#-------------------------------------------------
19611955
# make install related stuff
19621956
PREFIX ?= /usr/local

‎TARGETS‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,21 @@ cpp_library(
132132
"cache/clock_cache.cc",
133133
"cache/lru_cache.cc",
134134
"cache/sharded_cache.cc",
135+
"cloud/aws/aws_env.cc",
136+
"cloud/aws/aws_kafka.cc",
137+
"cloud/aws/aws_kinesis.cc",
138+
"cloud/aws/aws_retry.cc",
139+
"cloud/aws/aws_s3.cc",
140+
"cloud/cloud_env.cc",
141+
"cloud/cloud_env_impl.cc",
142+
"cloud/cloud_env_options.cc",
143+
"cloud/cloud_log_controller.cc",
144+
"cloud/cloud_manifest.cc",
145+
"cloud/cloud_scheduler.cc",
146+
"cloud/cloud_storage_provider.cc",
147+
"cloud/db_cloud_impl.cc",
148+
"cloud/manifest_reader.cc",
149+
"cloud/purge.cc",
135150
"db/arena_wrapped_db_iter.cc",
136151
"db/blob/blob_file_addition.cc",
137152
"db/blob/blob_file_builder.cc",
@@ -162,6 +177,7 @@ cpp_library(
162177
"db/db_impl/db_impl_files.cc",
163178
"db/db_impl/db_impl_open.cc",
164179
"db/db_impl/db_impl_readonly.cc",
180+
"db/db_impl/db_impl_remote_compaction.cc",
165181
"db/db_impl/db_impl_secondary.cc",
166182
"db/db_impl/db_impl_write.cc",
167183
"db/db_info_dumper.cc",
@@ -420,6 +436,21 @@ cpp_library(
420436
"cache/clock_cache.cc",
421437
"cache/lru_cache.cc",
422438
"cache/sharded_cache.cc",
439+
"cloud/aws/aws_env.cc",
440+
"cloud/aws/aws_kafka.cc",
441+
"cloud/aws/aws_kinesis.cc",
442+
"cloud/aws/aws_retry.cc",
443+
"cloud/aws/aws_s3.cc",
444+
"cloud/cloud_env.cc",
445+
"cloud/cloud_env_impl.cc",
446+
"cloud/cloud_env_options.cc",
447+
"cloud/cloud_log_controller.cc",
448+
"cloud/cloud_manifest.cc",
449+
"cloud/cloud_scheduler.cc",
450+
"cloud/cloud_storage_provider.cc",
451+
"cloud/db_cloud_impl.cc",
452+
"cloud/manifest_reader.cc",
453+
"cloud/purge.cc",
423454
"db/arena_wrapped_db_iter.cc",
424455
"db/blob/blob_file_addition.cc",
425456
"db/blob/blob_file_builder.cc",
@@ -450,6 +481,7 @@ cpp_library(
450481
"db/db_impl/db_impl_files.cc",
451482
"db/db_impl/db_impl_open.cc",
452483
"db/db_impl/db_impl_readonly.cc",
484+
"db/db_impl/db_impl_remote_compaction.cc",
453485
"db/db_impl/db_impl_secondary.cc",
454486
"db/db_impl/db_impl_write.cc",
455487
"db/db_info_dumper.cc",
@@ -976,6 +1008,20 @@ ROCKS_TESTS = [
9761008
[],
9771009
[],
9781010
],
1011+
[
1012+
"cloud_manifest_test",
1013+
"cloud/cloud_manifest_test.cc",
1014+
"serial",
1015+
[],
1016+
[],
1017+
],
1018+
[
1019+
"cloud_scheduler_test",
1020+
"cloud/cloud_scheduler_test.cc",
1021+
"serial",
1022+
[],
1023+
[],
1024+
],
9791025
[
9801026
"coding_test",
9811027
"util/coding_test.cc",
@@ -1116,6 +1162,13 @@ ROCKS_TESTS = [
11161162
[],
11171163
[],
11181164
],
1165+
[
1166+
"db_cloud_test",
1167+
"cloud/db_cloud_test.cc",
1168+
"serial",
1169+
[],
1170+
[],
1171+
],
11191172
[
11201173
"db_compaction_filter_test",
11211174
"db/db_compaction_filter_test.cc",
@@ -1767,6 +1820,13 @@ ROCKS_TESTS = [
17671820
[],
17681821
[],
17691822
],
1823+
[
1824+
"remote_compaction_test",
1825+
"cloud/remote_compaction_test.cc",
1826+
"serial",
1827+
[],
1828+
[],
1829+
],
17701830
[
17711831
"repair_test",
17721832
"db/repair_test.cc",

‎buckifier/buckify_rocksdb.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@ def get_cc_files(repo_path):
6464
continue
6565
for filename in fnmatch.filter(filenames, '*.cc'):
6666
cc_files.append(os.path.join(root, filename))
67+
for filename in fnmatch.filter(filenames, '*.c'):
68+
cc_files.append(os.path.join(root, filename))
6769
return cc_files
6870

6971

‎cloud/db_cloud_impl.cc‎

Lines changed: 18 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -132,27 +132,24 @@ Status DBCloud::Open(const Options& opt, const std::string& local_dbname,
132132
if (!persistent_cache_path.empty() && persistent_cache_size_gb) {
133133
// Get existing options. If the persistent cache is already set, then do
134134
// not make any change. Otherwise, configure it.
135-
void* bopt = options.table_factory->GetOptions();
136-
if (bopt != nullptr) {
137-
BlockBasedTableOptions* tableopt =
138-
static_cast<BlockBasedTableOptions*>(bopt);
139-
if (!tableopt->persistent_cache) {
140-
PersistentCacheConfig config(
141-
local_env, persistent_cache_path,
142-
persistent_cache_size_gb * 1024L * 1024L * 1024L, options.info_log);
143-
auto pcache = std::make_shared<BlockCacheTier>(config);
144-
st = pcache->Open();
145-
if (st.ok()) {
146-
tableopt->persistent_cache = pcache;
147-
Log(InfoLogLevel::INFO_LEVEL, options.info_log,
148-
"Created persistent cache %s with size %" PRIu64 "GB",
149-
persistent_cache_path.c_str(), persistent_cache_size_gb);
150-
} else {
151-
Log(InfoLogLevel::INFO_LEVEL, options.info_log,
152-
"Unable to create persistent cache %s. %s",
153-
persistent_cache_path.c_str(), st.ToString().c_str());
154-
return st;
155-
}
135+
auto* tableopt =
136+
options.table_factory->GetOptions<BlockBasedTableOptions>();
137+
if (tableopt != nullptr && !tableopt->persistent_cache) {
138+
PersistentCacheConfig config(
139+
local_env, persistent_cache_path,
140+
persistent_cache_size_gb * 1024L * 1024L * 1024L, options.info_log);
141+
auto pcache = std::make_shared<BlockCacheTier>(config);
142+
st = pcache->Open();
143+
if (st.ok()) {
144+
tableopt->persistent_cache = pcache;
145+
Log(InfoLogLevel::INFO_LEVEL, options.info_log,
146+
"Created persistent cache %s with size %" PRIu64 "GB",
147+
persistent_cache_path.c_str(), persistent_cache_size_gb);
148+
} else {
149+
Log(InfoLogLevel::INFO_LEVEL, options.info_log,
150+
"Unable to create persistent cache %s. %s",
151+
persistent_cache_path.c_str(), st.ToString().c_str());
152+
return st;
156153
}
157154
}
158155
}

‎db/blob/blob_log_reader.cc‎

Lines changed: 0 additions & 134 deletions
This file was deleted.

0 commit comments

Comments
 (0)