Upgrade rocksdb

Signed-off-by: Yang Zhang <[email protected]>

Author: v01dstar

.gitmodules

@@ -1,9 +1,9 @@
 [submodule "rocksdb"]
 	path = librocksdb_sys/rocksdb
-	url = https://github.com/tikv/rocksdb.git
-	branch = 6.29.tikv
+	url = https://github.com/v01dstar/rocksdb.git
+	branch = 8.10-tikv
 
 [submodule "titan"]
 	path = librocksdb_sys/libtitan_sys/titan
-	url = https://github.com/tikv/titan.git
-	branch = master
+	url = https://github.com/v01dstar/titan.git
+	branch = upgrade-rocksdb

librocksdb_sys/build.rs

@@ -76,7 +76,7 @@ fn main() {
 
     build.cpp(true).file("crocksdb/c.cc");
     if env::var("CARGO_CFG_TARGET_OS").unwrap() != "windows" {
-        build.flag("-std=c++11");
+        build.flag("-std=c++17");
         build.flag("-fno-rtti");
     }
     link_cpp(&mut build);

librocksdb_sys/crocksdb/c.cc

@@ -48,7 +48,7 @@
 #include "rocksdb/table_properties.h"
 #include "rocksdb/types.h"
 #include "rocksdb/universal_compaction.h"
-#include "rocksdb/utilities/backupable_db.h"
+#include "rocksdb/utilities/backup_engine.h"
 #include "rocksdb/utilities/checkpoint.h"
 #include "rocksdb/utilities/db_ttl.h"
 #include "rocksdb/utilities/debug.h"
@@ -75,8 +75,8 @@
 #endif
 
 using rocksdb::BackgroundErrorReason;
-using rocksdb::BackupableDBOptions;
 using rocksdb::BackupEngine;
+using rocksdb::BackupEngineOptions;
 using rocksdb::BackupInfo;
 using rocksdb::BlockBasedTableOptions;
 using rocksdb::BlockCipher;
@@ -524,41 +524,10 @@ struct crocksdb_filterpolicy_t : public FilterPolicy {
   void* state_;
   void (*destructor_)(void*);
   const char* (*name_)(void*);
-  char* (*create_)(void*, const char* const* key_array,
-                   const size_t* key_length_array, int num_keys,
-                   size_t* filter_length);
-  unsigned char (*key_match_)(void*, const char* key, size_t length,
-                              const char* filter, size_t filter_length);
-  void (*delete_filter_)(void*, const char* filter, size_t filter_length);
 
   virtual ~crocksdb_filterpolicy_t() { (*destructor_)(state_); }
 
   virtual const char* Name() const override { return (*name_)(state_); }
-
-  virtual void CreateFilter(const Slice* keys, int n,
-                            std::string* dst) const override {
-    std::vector<const char*> key_pointers(n);
-    std::vector<size_t> key_sizes(n);
-    for (int i = 0; i < n; i++) {
-      key_pointers[i] = keys[i].data();
-      key_sizes[i] = keys[i].size();
-    }
-    size_t len;
-    char* filter = (*create_)(state_, &key_pointers[0], &key_sizes[0], n, &len);
-    dst->append(filter, len);
-
-    if (delete_filter_ != nullptr) {
-      (*delete_filter_)(state_, filter, len);
-    } else {
-      free(filter);
-    }
-  }
-
-  virtual bool KeyMayMatch(const Slice& key,
-                           const Slice& filter) const override {
-    return (*key_match_)(state_, key.data(), key.size(), filter.data(),
-                         filter.size());
-  }
 };
 
 struct crocksdb_mergeoperator_t : public MergeOperator {
@@ -853,7 +822,7 @@ crocksdb_backup_engine_t* crocksdb_backup_engine_open(
     const crocksdb_options_t* options, const char* path, char** errptr) {
   BackupEngine* be;
   if (SaveError(errptr, BackupEngine::Open(options->rep.env,
-                                           BackupableDBOptions(path), &be))) {
+                                           BackupEngineOptions(path), &be))) {
     return nullptr;
   }
   crocksdb_backup_engine_t* result = new crocksdb_backup_engine_t;
@@ -1189,7 +1158,8 @@ void crocksdb_write_multi_batch(crocksdb_t* db,
   for (size_t i = 0; i < batch_size; i++) {
     ws.push_back(&batches[i]->rep);
   }
-  SaveError(errptr, db->rep->MultiBatchWrite(options->rep, std::move(ws)));
+  SaveError(errptr,
+            db->rep->MultiBatchWrite(options->rep, std::move(ws), nullptr));
 }
 
 void crocksdb_write_multi_batch_callback(
@@ -2097,14 +2067,6 @@ void crocksdb_block_based_options_set_block_cache(
   }
 }
 
-void crocksdb_block_based_options_set_block_cache_compressed(
-    crocksdb_block_based_table_options_t* options,
-    crocksdb_cache_t* block_cache_compressed) {
-  if (block_cache_compressed) {
-    options->rep.block_cache_compressed = block_cache_compressed->rep;
-  }
-}
-
 void crocksdb_block_based_options_set_whole_key_filtering(
     crocksdb_block_based_table_options_t* options, unsigned char v) {
   options->rep.whole_key_filtering = v;
@@ -2120,11 +2082,6 @@ void crocksdb_block_based_options_set_index_type(
   options->rep.index_type = static_cast<BlockBasedTableOptions::IndexType>(v);
 }
 
-void crocksdb_block_based_options_set_hash_index_allow_collision(
-    crocksdb_block_based_table_options_t* options, unsigned char v) {
-  options->rep.hash_index_allow_collision = v;
-}
-
 void crocksdb_block_based_options_set_optimize_filters_for_memory(
     crocksdb_block_based_table_options_t* options, unsigned char v) {
   options->rep.optimize_filters_for_memory = v;
@@ -3068,11 +3025,6 @@ void crocksdb_options_set_is_fd_close_on_exec(crocksdb_options_t* opt,
   opt->rep.is_fd_close_on_exec = v;
 }
 
-void crocksdb_options_set_skip_log_error_on_recovery(crocksdb_options_t* opt,
-                                                     unsigned char v) {
-  opt->rep.skip_log_error_on_recovery = v;
-}
-
 void crocksdb_options_set_stats_dump_period_sec(crocksdb_options_t* opt,
                                                 unsigned int v) {
   opt->rep.stats_dump_period_sec = v;
@@ -3197,16 +3149,6 @@ int crocksdb_options_get_max_background_compactions(
   return opt->rep.max_background_compactions;
 }
 
-void crocksdb_options_set_base_background_compactions(crocksdb_options_t* opt,
-                                                      int n) {
-  opt->rep.base_background_compactions = n;
-}
-
-int crocksdb_options_get_base_background_compactions(
-    const crocksdb_options_t* opt) {
-  return opt->rep.base_background_compactions;
-}
-
 void crocksdb_options_set_max_background_flushes(crocksdb_options_t* opt,
                                                  int n) {
   opt->rep.max_background_flushes = n;
@@ -3234,14 +3176,6 @@ void crocksdb_options_set_recycle_log_file_num(crocksdb_options_t* opt,
   opt->rep.recycle_log_file_num = v;
 }
 
-void crocksdb_options_set_soft_rate_limit(crocksdb_options_t* opt, double v) {
-  opt->rep.soft_rate_limit = v;
-}
-
-void crocksdb_options_set_hard_rate_limit(crocksdb_options_t* opt, double v) {
-  opt->rep.hard_rate_limit = v;
-}
-
 void crocksdb_options_set_soft_pending_compaction_bytes_limit(
     crocksdb_options_t* opt, size_t v) {
   opt->rep.soft_pending_compaction_bytes_limit = v;
@@ -3262,11 +3196,6 @@ size_t crocksdb_options_get_hard_pending_compaction_bytes_limit(
   return opt->rep.hard_pending_compaction_bytes_limit;
 }
 
-void crocksdb_options_set_rate_limit_delay_max_milliseconds(
-    crocksdb_options_t* opt, unsigned int v) {
-  opt->rep.rate_limit_delay_max_milliseconds = v;
-}
-
 void crocksdb_options_set_max_manifest_file_size(crocksdb_options_t* opt,
                                                  size_t v) {
   opt->rep.max_manifest_file_size = v;
@@ -3609,12 +3538,14 @@ void crocksdb_options_set_track_and_verify_wals_in_manifest(
 }
 
 unsigned char crocksdb_load_latest_options(
-    const char* dbpath, crocksdb_env_t* env, crocksdb_options_t* db_options,
+    const char* dbpath, crocksdb_options_t* db_options,
     crocksdb_column_family_descriptor*** cf_descs, size_t* cf_descs_len,
     unsigned char ignore_unknown_options, char** errptr) {
   std::vector<ColumnFamilyDescriptor> tmp_cf_descs;
-  Status s = rocksdb::LoadLatestOptions(dbpath, env->rep, &db_options->rep,
-                                        &tmp_cf_descs, ignore_unknown_options);
+  rocksdb::ConfigOptions config_options;
+  config_options.ignore_unknown_options = ignore_unknown_options;
+  Status s = rocksdb::LoadLatestOptions(rocksdb::ConfigOptions(), dbpath,
+                                        &db_options->rep, &tmp_cf_descs);
 
   *errptr = nullptr;
   if (s.IsNotFound()) return false;
@@ -3810,17 +3741,12 @@ unsigned char crocksdb_compactionfiltercontext_is_bottommost_level(
   return context->rep.is_bottommost_level;
 }
 
-void crocksdb_compactionfiltercontext_file_numbers(
-    crocksdb_compactionfiltercontext_t* context, const uint64_t** buffer,
-    size_t* len) {
-  *buffer = context->rep.file_numbers.data();
-  *len = context->rep.file_numbers.size();
-}
-
-crocksdb_table_properties_t* crocksdb_compactionfiltercontext_table_properties(
-    crocksdb_compactionfiltercontext_t* context, size_t offset) {
-  return (crocksdb_table_properties_t*)context->rep.table_properties[offset]
-      .get();
+crocksdb_table_properties_collection_t*
+crocksdb_compactionfiltercontext_input_table_properties(
+    crocksdb_compactionfiltercontext_t* context) {
+  return (
+      crocksdb_table_properties_collection_t*)(&context->rep
+                                                    .input_table_properties);
 }
 
 const char* crocksdb_compactionfiltercontext_start_key(
@@ -3879,25 +3805,6 @@ crocksdb_comparator_t* crocksdb_comparator_create(
 
 void crocksdb_comparator_destroy(crocksdb_comparator_t* cmp) { delete cmp; }
 
-crocksdb_filterpolicy_t* crocksdb_filterpolicy_create(
-    void* state, void (*destructor)(void*),
-    char* (*create_filter)(void*, const char* const* key_array,
-                           const size_t* key_length_array, int num_keys,
-                           size_t* filter_length),
-    unsigned char (*key_may_match)(void*, const char* key, size_t length,
-                                   const char* filter, size_t filter_length),
-    void (*delete_filter)(void*, const char* filter, size_t filter_length),
-    const char* (*name)(void*)) {
-  crocksdb_filterpolicy_t* result = new crocksdb_filterpolicy_t;
-  result->state_ = state;
-  result->destructor_ = destructor;
-  result->create_ = create_filter;
-  result->key_match_ = key_may_match;
-  result->delete_filter_ = delete_filter;
-  result->name_ = name;
-  return result;
-}
-
 void crocksdb_filterpolicy_destroy(crocksdb_filterpolicy_t* filter) {
   delete filter;
 }
@@ -3910,11 +3817,8 @@ struct FilterPolicyWrapper : public crocksdb_filterpolicy_t {
   std::string full_name_;
   ~FilterPolicyWrapper() override { delete rep_; }
   const char* Name() const override { return full_name_.c_str(); }
-  void CreateFilter(const Slice* keys, int n, std::string* dst) const override {
-    return rep_->CreateFilter(keys, n, dst);
-  }
-  bool KeyMayMatch(const Slice& key, const Slice& filter) const override {
-    return rep_->KeyMayMatch(key, filter);
+  const char* CompatibilityName() const override {
+    return rep_->CompatibilityName();
   }
   // No need to override GetFilterBitsBuilder if this one is overridden
   FilterBitsBuilder* GetBuilderWithContext(
@@ -3938,7 +3842,6 @@ crocksdb_filterpolicy_t* crocksdb_filterpolicy_create_bloom_format(
     wrapper->full_name_ += ".FullBloom";
   }
   wrapper->state_ = nullptr;
-  wrapper->delete_filter_ = nullptr;
   wrapper->destructor_ = &FilterPolicyWrapper::DoNothing;
   return wrapper;
 }
@@ -3961,7 +3864,6 @@ crocksdb_filterpolicy_t* crocksdb_filterpolicy_create_ribbon(
   wrapper->full_name_ = wrapper->rep_->Name();
   wrapper->full_name_ += ".Ribbon";
   wrapper->state_ = nullptr;
-  wrapper->delete_filter_ = nullptr;
   wrapper->destructor_ = &FilterPolicyWrapper::DoNothing;
   return wrapper;
 }
@@ -5821,7 +5723,7 @@ struct ExternalSstFileModifier {
     auto ioptions = *cfd->ioptions();
     auto table_opt =
         TableReaderOptions(ioptions, desc.options.prefix_extractor,
-                           env_options_, cfd->internal_comparator());
+                           env_options_, cfd->internal_comparator(), 0);
     // Get around global seqno check.
     table_opt.largest_seqno = kMaxSequenceNumber;
     status = ioptions.table_factory->NewTableReader(

librocksdb_sys/crocksdb/crocksdb/c.h

@@ -780,10 +780,6 @@ extern C_ROCKSDB_LIBRARY_API void crocksdb_block_based_options_set_block_cache(
     crocksdb_block_based_table_options_t* options,
     crocksdb_cache_t* block_cache);
 extern C_ROCKSDB_LIBRARY_API void
-crocksdb_block_based_options_set_block_cache_compressed(
-    crocksdb_block_based_table_options_t* options,
-    crocksdb_cache_t* block_cache_compressed);
-extern C_ROCKSDB_LIBRARY_API void
 crocksdb_block_based_options_set_whole_key_filtering(
     crocksdb_block_based_table_options_t*, unsigned char);
 extern C_ROCKSDB_LIBRARY_API void
@@ -1167,7 +1163,7 @@ crocksdb_options_get_sst_partitioner_factory(crocksdb_options_t*);
 extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_sst_partitioner_factory(
     crocksdb_options_t*, crocksdb_sst_partitioner_factory_t*);
 extern C_ROCKSDB_LIBRARY_API unsigned char crocksdb_load_latest_options(
-    const char* dbpath, crocksdb_env_t* env, crocksdb_options_t* db_options,
+    const char* dbpath, crocksdb_options_t* db_options,
     crocksdb_column_family_descriptor*** cf_descs, size_t* cf_descs_len,
     unsigned char ignore_unknown_options, char** errptr);
 
@@ -1222,10 +1218,6 @@ extern C_ROCKSDB_LIBRARY_API void
 crocksdb_options_set_max_background_compactions(crocksdb_options_t*, int);
 extern C_ROCKSDB_LIBRARY_API int
 crocksdb_options_get_max_background_compactions(const crocksdb_options_t*);
-extern C_ROCKSDB_LIBRARY_API void
-crocksdb_options_set_base_background_compactions(crocksdb_options_t*, int);
-extern C_ROCKSDB_LIBRARY_API int
-crocksdb_options_get_base_background_compactions(const crocksdb_options_t*);
 extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_max_background_flushes(
     crocksdb_options_t*, int);
 extern C_ROCKSDB_LIBRARY_API int crocksdb_options_get_max_background_flushes(
@@ -1238,10 +1230,6 @@ extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_keep_log_file_num(
     crocksdb_options_t*, size_t);
 extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_recycle_log_file_num(
     crocksdb_options_t*, size_t);
-extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_soft_rate_limit(
-    crocksdb_options_t*, double);
-extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_hard_rate_limit(
-    crocksdb_options_t*, double);
 extern C_ROCKSDB_LIBRARY_API void
 crocksdb_options_set_soft_pending_compaction_bytes_limit(
     crocksdb_options_t* opt, size_t v);
@@ -1254,9 +1242,6 @@ crocksdb_options_set_hard_pending_compaction_bytes_limit(
 extern C_ROCKSDB_LIBRARY_API size_t
 crocksdb_options_get_hard_pending_compaction_bytes_limit(
     crocksdb_options_t* opt);
-extern C_ROCKSDB_LIBRARY_API void
-crocksdb_options_set_rate_limit_delay_max_milliseconds(crocksdb_options_t*,
-                                                       unsigned int);
 extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_max_manifest_file_size(
     crocksdb_options_t*, size_t);
 extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_table_cache_numshardbits(
@@ -1293,9 +1278,6 @@ extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_allow_mmap_writes(
     crocksdb_options_t*, unsigned char);
 extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_is_fd_close_on_exec(
     crocksdb_options_t*, unsigned char);
-extern C_ROCKSDB_LIBRARY_API void
-crocksdb_options_set_skip_log_error_on_recovery(crocksdb_options_t*,
-                                                unsigned char);
 extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_stats_dump_period_sec(
     crocksdb_options_t*, unsigned int);
 extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_stats_persist_period_sec(
@@ -1549,13 +1531,9 @@ extern C_ROCKSDB_LIBRARY_API unsigned char
 crocksdb_compactionfiltercontext_is_bottommost_level(
     crocksdb_compactionfiltercontext_t* context);
 
-extern C_ROCKSDB_LIBRARY_API void crocksdb_compactionfiltercontext_file_numbers(
-    crocksdb_compactionfiltercontext_t* context, const uint64_t** buffer,
-    size_t* len);
-
-extern C_ROCKSDB_LIBRARY_API crocksdb_table_properties_t*
-crocksdb_compactionfiltercontext_table_properties(
-    crocksdb_compactionfiltercontext_t* context, size_t offset);
+extern C_ROCKSDB_LIBRARY_API crocksdb_table_properties_collection_t*
+crocksdb_compactionfiltercontext_input_table_properties(
+    crocksdb_compactionfiltercontext_t* context);
 
 extern C_ROCKSDB_LIBRARY_API uint32_t crocksdb_compactionfiltercontext_reason(
     crocksdb_compactionfiltercontext_t* context);
@@ -1584,16 +1562,6 @@ extern C_ROCKSDB_LIBRARY_API void crocksdb_comparator_destroy(
 
 /* Filter policy */
 
-extern C_ROCKSDB_LIBRARY_API crocksdb_filterpolicy_t*
-crocksdb_filterpolicy_create(
-    void* state, void (*destructor)(void*),
-    char* (*create_filter)(void*, const char* const* key_array,
-                           const size_t* key_length_array, int num_keys,
-                           size_t* filter_length),
-    unsigned char (*key_may_match)(void*, const char* key, size_t length,
-                                   const char* filter, size_t filter_length),
-    void (*delete_filter)(void*, const char* filter, size_t filter_length),
-    const char* (*name)(void*));
 extern C_ROCKSDB_LIBRARY_API void crocksdb_filterpolicy_destroy(
     crocksdb_filterpolicy_t*);