Merge #12604: Add DynamicMemoryUsage() to CDBWrapper to estimate LevelDB memory use

741f017 Add DynamicMemoryUsage() to LevelDB (Evan Klitzke)

Pull request description:

  This adds a new method `CDBWrapper::DynamicMemoryUsage()` similar to Bitcoin's existing methods of the same name. It's implemented by asking LevelDB for the information, and then parsing the string response. I've also added logging to `CDBWrapper::WriteBatch()` to track this information:

  ```
  $ tail -f ~/.bitcoin/testnet3/debug.log | grep WriteBatch
  2018-03-05 19:34:55 WriteBatch memory usage: db=chainstate, before=0.0MiB, after=0.0MiB
  2018-03-05 19:35:17 WriteBatch memory usage: db=index, before=0.0MiB, after=0.0MiB
  2018-03-05 19:35:17 WriteBatch memory usage: db=chainstate, before=0.0MiB, after=8.0MiB
  2018-03-05 19:35:22 WriteBatch memory usage: db=index, before=0.0MiB, after=0.0MiB
  2018-03-05 19:35:22 WriteBatch memory usage: db=chainstate, before=8.0MiB, after=17.0MiB
  2018-03-05 19:35:26 WriteBatch memory usage: db=index, before=0.0MiB, after=0.0MiB
  2018-03-05 19:35:27 WriteBatch memory usage: db=chainstate, before=9.0MiB, after=18.0MiB
  2018-03-05 19:35:40 WriteBatch memory usage: db=index, before=0.0MiB, after=0.0MiB
  2018-03-05 19:35:41 WriteBatch memory usage: db=chainstate, before=9.0MiB, after=7.0MiB
  2018-03-05 19:35:52 WriteBatch memory usage: db=index, before=0.0MiB, after=0.0MiB
  2018-03-05 19:35:52 WriteBatch memory usage: db=chainstate, before=7.0MiB, after=9.0MiB
  ^C
  ```

  As LevelDB doesn't seem to provide a way to get the database name, I've also added a new `m_name` field to the `CDBWrapper`. This is necessary because we have multiple LevelDB databases (two now, and possibly more later, e.g. #11857).

  I am using this information in other branches where I'm experimenting with changing LevelDB buffer sizes.

Tree-SHA512: 7ea8ff5484bb07ef806af17d000c74ccca27d2e0f6c3229e12d93818f00874553335d87428482bd8acbcae81ea35aef2a243326f9fccbfac25989323d24391b4

Author: laanwj

src/dbwrapper.cpp

@@ -89,6 +89,7 @@ static leveldb::Options GetOptions(size_t nCacheSize)
 }
 
 CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate)
+    : m_name(fs::basename(path))
 {
     penv = nullptr;
     readoptions.verify_checksums = true;
@@ -155,11 +156,30 @@ CDBWrapper::~CDBWrapper()
 
 bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync)
 {
+    const bool log_memory = LogAcceptCategory(BCLog::LEVELDB);
+    double mem_before = 0;
+    if (log_memory) {
+        mem_before = DynamicMemoryUsage() / 1024 / 1024;
+    }
     leveldb::Status status = pdb->Write(fSync ? syncoptions : writeoptions, &batch.batch);
     dbwrapper_private::HandleError(status);
+    if (log_memory) {
+        double mem_after = DynamicMemoryUsage() / 1024 / 1024;
+        LogPrint(BCLog::LEVELDB, "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n",
+                 m_name, mem_before, mem_after);
+    }
     return true;
 }
 
+size_t CDBWrapper::DynamicMemoryUsage() const {
+    std::string memory;
+    if (!pdb->GetProperty("leveldb.approximate-memory-usage", &memory)) {
+        LogPrint(BCLog::LEVELDB, "Failed to get approximate-memory-usage property\n");
+        return 0;
+    }
+    return stoul(memory);
+}
+
 // Prefixed with null character to avoid collisions with other keys
 //
 // We must use a string constructor which specifies length so that we copy

src/dbwrapper.h

@@ -198,6 +198,9 @@ class CDBWrapper
     //! the database itself
     leveldb::DB* pdb;
 
+    //! the name of this database
+    std::string m_name;
+
     //! a key used for optional XOR-obfuscation of the database
     std::vector<unsigned char> obfuscate_key;
 
@@ -284,6 +287,9 @@ class CDBWrapper
 
     bool WriteBatch(CDBBatch& batch, bool fSync = false);
 
+    // Get an estimate of LevelDB memory usage (in bytes).
+    size_t DynamicMemoryUsage() const;
+
     // not available for LevelDB; provide for compatibility with BDB
     bool Flush()
     {