Use ToplingDB step by step

[rockeet]

From ToplingDB document

中文版 (Chinese)

Using ToplingDB from scratch

Sample code (error handling omitted, full version here)

To migrate from RocksDB to ToplingDB, you only need to modify the code related to Open/Close, and no other code needs to be changed.

#include <topling/side_plugin_repo.h>
#include <rocksdb/db.h>
int main(int argc, char* argv[]) {
  using namespace rocksdb;
  SidePluginRepo repo;
  repo.ImportAutoFile(argv[1]); // argv[1] is json/yaml conf file
  DB* db = nullptr;
  repo.OpenDB(&db);
  repo.StartHttpServer();
  if (strcasecmp(argv[2], "set") == 0) {
    db->Put(WriteOptions(), argv[3], argv[4]);
  }
  else if (strcasecmp(argv[2], "get") == 0) {
    std::string val;
    db->Get(ReadOptions(), argv[3], &val);
    fprintf(stdout, "%s\n", val.c_str());
  }
  fprintf(stderr, "now visit the web(defined in json/yaml conf file)\n");
  fprintf(stderr, "press enter to exit\n");
  getchar(); // wait for enter
  repo.CloseAllDB();
  return 0;
}

1. Open

SidePluginRepo is the core class of ToplingDB SidePlugin, and all components are included in it. To use SidePlugin, you must first define a SidePluginRepo object:

  SidePluginRepo repo;

With the SidePluginRepo object, the next step is to import configuration parameters. The most convenient way is to call ImportAutoFile, which automatically recognizes json and yaml files according to the file extension:

  repo.ImportAutoFile(argv[1]); // argv[1] is json/yaml conf file

Now, we can open the DB:

  DB* db = nullptr;
  repo.OpenDB(&db);

In RocksDB, we generally use DB::Open, and in a few cases use other Open functions. But in SidePlugin, we use OpenDB uniformly, and which Open function is used is configured in json/yaml: for example Todis secondary node.

In this example, we use the default DB with only one ColumnFamily, but in fact OpenDB is an overloaded function, which can also be used to open a DB with multiple ColumnFamily (db_bench modified later is an example of this):

  DB_MultiCF* dbmcf = nullptr;
  repo.OpenDB(&dbmcf); // overload function name

2. Start the embedded Http Web Server

In theory, the embedded Http Web Server can be automatically started in OpenDB. However, we still decided to let the user start it explicitly:

  repo.StartHttpServer();

There are three main reasons:

1. Users can explicitly open multiple DBs defined in the same json/yaml, but there is only one embedded Http
2. With the interface style of RocksDB, error handling uses the return value Status, it is difficult to use the same Status to distinguish the error information of DB Open and Http Start
3. Even if http is defined in json/yaml, users may not want to enable the embedded Http Web Server

Close

Need explicit Close, just one line of code:

  repo.CloseAllDB();

Further reference: Using CompactionFilter in ToplingDB

---

A bit more complicated: db_bench modification for ToplingDB

db_bench_tool.cc is a file with more than 8000 lines. We make it support ToplingDB, adding only 47 lines of code, without any other modification, then we have all the capabilities of ToplingDB! Here is the diff content

diff --git a/tools/db_bench_tool.cc b/tools/db_bench_tool.cc
index fcd3c157a..3a1f632a5 100644
--- a/tools/db_bench_tool.cc
+++ b/tools/db_bench_tool.cc
@@ -97,6 +97,8 @@
 #include <io.h>  // open/close
 #endif

+#include "sideplugin/rockside/src/topling/side_plugin_repo.h"
+
 using GFLAGS_NAMESPACE::ParseCommandLineFlags;
 using GFLAGS_NAMESPACE::RegisterFlagValidator;
 using GFLAGS_NAMESPACE::SetUsageMessage;
@@ -1052,6 +1054,7 @@ DEFINE_int32(trace_replay_threads, 1,
 DEFINE_bool(io_uring_enabled, true,
             "If true, enable the use of IO uring if the platform supports it");
 extern "C" bool RocksDbIOUringEnable() { return FLAGS_io_uring_enabled; }
+DEFINE_string(json, "", "json config file.");
 #endif  // ROCKSDB_LITE

 DEFINE_bool(adaptive_readahead, false,
@@ -3087,6 +3090,7 @@ class Benchmark {
   }

   void DeleteDBs() {
+    repo_.CloseAllDB(false);
     db_.DeleteDBs();
     for (const DBWithColumnFamilies& dbwcf : multi_dbs_) {
       delete dbwcf.db;
@@ -3104,6 +3108,11 @@ class Benchmark {
     }
   }

+  void exit(int code) {
+    this->~Benchmark();
+    ::exit(code);
+  }
+
   Slice AllocateKey(std::unique_ptr<const char[]>* key_guard) {
     char* data = new char[key_size_];
     const char* const_data = data;
@@ -3236,6 +3245,7 @@ class Benchmark {
       ErrorExit();
     }
     Open(&open_options_);
+    open_options_ = db_.db->GetOptions();
     PrintHeader(open_options_);
     std::stringstream benchmark_stream(FLAGS_benchmarks);
     std::string name;
@@ -4516,9 +4526,45 @@ class Benchmark {
     InitializeOptionsGeneral(opts);
   }

+  SidePluginRepo repo_;
   void OpenDb(Options options, const std::string& db_name,
       DBWithColumnFamilies* db) {
     uint64_t open_start = FLAGS_report_open_timing ? FLAGS_env->NowNanos() : 0;
+    if (!FLAGS_json.empty()) {
+      repo_.CloseAllDB(false);
+      repo_.CleanResetRepo();
+      DB_MultiCF* dbmcf = nullptr;
+      Status s = repo_.ImportAutoFile(FLAGS_json);
+      if (!s.ok()) {
+        fprintf(stderr, "ERROR: ImportAutoFile(%s): %s\n",
+                FLAGS_json.c_str(), s.ToString().c_str());
+        exit(1);
+      }
+      s = repo_.OpenDB(&dbmcf);
+      if (!s.ok()) {
+        fprintf(stderr, "ERROR: OpenDB(): Config File=%s: %s\n",
+                FLAGS_json.c_str(), s.ToString().c_str());
+        exit(1);
+      }
+      s = repo_.StartHttpServer();
+      if (!s.ok()) {
+        fprintf(stderr, "ERROR: StartHttpServer(): JsonFile=%s: %s\n",
+                FLAGS_json.c_str(), s.ToString().c_str());
+        exit(1);
+      }
+      db->cfh = dbmcf->cf_handles;
+      db->db = dbmcf->db;
+      if (auto tdb = dynamic_cast<OptimisticTransactionDB*>(dbmcf->db)) {
+        db->opt_txn_db = tdb;
+        db->db = tdb->GetBaseDB();
+      }
+      db->num_created = FLAGS_num_column_families;
+      db->num_hot = FLAGS_num_column_families;
+      DBOptions dbo = db->db->GetDBOptions();
+      dbstats = dbo.statistics;
+      FLAGS_db = db->db->GetName();
+      return;
+    }
     Status s;
     // Open with column families if necessary.
     if (FLAGS_num_column_families > 1) {