MemLens Database Specific Changes (#170)

* global and local build config with profiling flags.

* integrated basic memory hook for RSS calculation

* edited script to generate callgraph using gprof

* enabling level db as default storage engine

* systemd telegraf script to stream prometheus node_exporter data to InfluxDB.

* added lmdb storage engine interface and partially implemented.

* bootstrapping script to run perf tools

* adding process exporter to bootstrapping script

* Added LRU Cache and Observabiity stats

* lry cache controlled using settings

* removing lmdb implementation

* removing unecessary scripts

* Resolving comments for build flags and memory leak

* resolved error. options.block_cache_ integration still not working

* added UT's for TC's. removed logs from stats and changed lru implementation var names

* reverting configs and removing unwanted files

* removing unwanted tst logs

* refactor: rename GetMetrics to UpdateMetrics and improve cache handling in ResLevelDB

* feat: add LevelDB block cache support and corresponding tests.

* feat: enhance LevelDB block cache with configurable capacity and update metrics return type

* refactor: update transaction summary variable names for consistency

* build: optimize compilation flags by adding optimization level

* build: remove debugging flags from kv_service and api_tools builds

* build: remove profiling flags from set_random_data binary

* build: remove pyroscope monitoring from kv service startup script

* build: include string header in lru_cache implementation

Author: harish876

.bazelrc

@@ -1,4 +1,4 @@
-build --cxxopt='-std=c++17' --copt=-O3 --jobs=40 
+build --cxxopt='-std=c++17' --copt=-O3 --jobs=40
 #build --action_env=PYTHON_BIN_PATH="/usr/bin/python3.10"
 #build --action_env=PYTHON_LIB_PATH="/usr/include/python3.10"
 

.gitignore

@@ -13,3 +13,10 @@ sdk_validator/venv
 __pycache__
 MODULE.*
 apache_release
+*.out.*
+*.data.*
+*.pb.*
+.cache/
+resdb/
+100*_db/
+gmon.out

WORKSPACE

@@ -1,6 +1,22 @@
 workspace(name = "com_resdb_nexres")
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "hedron_compile_commands",
+    #Replace the commit hash (4f28899228fb3ad0126897876f147ca15026151e) with the latest commit hash from the repo
+    url = "https://github.com/hedronvision/bazel-compile-commands-extractor/archive/4f28899228fb3ad0126897876f147ca15026151e.tar.gz",
+    strip_prefix = "bazel-compile-commands-extractor-4f28899228fb3ad0126897876f147ca15026151e",
+)
+load("@hedron_compile_commands//:workspace_setup.bzl", "hedron_compile_commands_setup")
+hedron_compile_commands_setup()
+load("@hedron_compile_commands//:workspace_setup_transitive.bzl", "hedron_compile_commands_setup_transitive")
+hedron_compile_commands_setup_transitive()
+load("@hedron_compile_commands//:workspace_setup_transitive_transitive.bzl", "hedron_compile_commands_setup_transitive_transitive")
+hedron_compile_commands_setup_transitive_transitive()
+load("@hedron_compile_commands//:workspace_setup_transitive_transitive_transitive.bzl", "hedron_compile_commands_setup_transitive_transitive_transitive")
+hedron_compile_commands_setup_transitive_transitive_transitive()
+
 load("//:repositories.bzl", "nexres_repositories")
 
 nexres_repositories()

chain/storage/BUILD

@@ -52,6 +52,8 @@ cc_library(
         "//chain/storage/proto:kv_cc_proto",
         "//chain/storage/proto:leveldb_config_cc_proto",
         "//common:comm",
+        "//common/lru:lru_cache",
+        "//platform/statistic:stats",
         "//third_party:leveldb",
     ],
 )
@@ -64,4 +66,18 @@ cc_test(
         ":memory_db",
         "//common/test:test_main",
     ],
+    timeout = "short",  # Set the timeout to "short"
+    size = "small",     # Set the size to "small"
 )
+
+cc_test(
+    name = "leveldb_test",
+    srcs = ["leveldb_test.cpp"],
+    deps = [
+        ":leveldb",
+        "//platform/statistic:stats",
+        "//common/test:test_main",
+    ],
+    timeout = "short",  # Set the timeout to "short"
+    size = "small",     # Set the size to "small"
+)

chain/storage/kv_storage_test.cpp

@@ -30,10 +30,7 @@ namespace resdb {
 namespace storage {
 namespace {
 
-enum StorageType {
-  MEM = 0,
-  LEVELDB = 1,
-};
+enum StorageType { MEM = 0, LEVELDB = 1, LEVELDB_WITH_BLOCK_CACHE = 2 };
 
 class KVStorageTest : public ::testing::TestWithParam<StorageType> {
  protected:
@@ -47,6 +44,12 @@ class KVStorageTest : public ::testing::TestWithParam<StorageType> {
         Reset();
         storage = NewResLevelDB(path_);
         break;
+      case LEVELDB_WITH_BLOCK_CACHE:
+        Reset();
+        LevelDBInfo config;
+        config.set_enable_block_cache(true);
+        storage = NewResLevelDB(path_, config);
+        break;
     }
   }
 
@@ -218,8 +221,16 @@ TEST_P(KVStorageTest, GetHistory) {
   }
 }
 
+TEST_P(KVStorageTest, BlockCacheSpecificTest) {
+  if (GetParam() == LEVELDB_WITH_BLOCK_CACHE) {
+    std::cout << "Running BlockCacheSpecificTest for LEVELDB_WITH_BLOCK_CACHE"
+              << std::endl;
+  }
+}
+
 INSTANTIATE_TEST_CASE_P(KVStorageTest, KVStorageTest,
-                        ::testing::Values(MEM, LEVELDB));
+                        ::testing::Values(MEM, LEVELDB,
+                                          LEVELDB_WITH_BLOCK_CACHE));
 
 }  // namespace
 }  // namespace storage

chain/storage/leveldb.cpp

@@ -20,8 +20,12 @@
 #include "chain/storage/leveldb.h"
 
 #include <glog/logging.h>
+#include <unistd.h>
+
+#include <cstdint>
 
 #include "chain/storage/proto/kv.pb.h"
+#include "leveldb/options.h"
 
 namespace resdb {
 namespace storage {
@@ -50,6 +54,16 @@ ResLevelDB::ResLevelDB(std::optional<LevelDBInfo> config) {
       path = (*config).path();
     }
   }
+  if ((*config).enable_block_cache()) {
+    uint32_t capacity = 1000;
+    if ((*config).has_block_cache_capacity()) {
+      capacity = (*config).block_cache_capacity();
+    }
+    block_cache_ =
+        std::make_unique<LRUCache<std::string, std::string>>(capacity);
+    LOG(ERROR) << "initialized block cache" << std::endl;
+  }
+  global_stats_ = Stats::GetGlobalStats();
   CreateDB(path);
 }
 
@@ -74,15 +88,22 @@ ResLevelDB::~ResLevelDB() {
   if (db_) {
     db_.reset();
   }
+  if (block_cache_) {
+    block_cache_->Flush();
+  }
 }
 
 int ResLevelDB::SetValue(const std::string& key, const std::string& value) {
+  if (block_cache_) {
+    block_cache_->Put(key, value);
+  }
   batch_.Put(key, value);
 
   if (batch_.ApproximateSize() >= write_batch_size_) {
     leveldb::Status status = db_->Write(leveldb::WriteOptions(), &batch_);
     if (status.ok()) {
       batch_.Clear();
+      UpdateMetrics();
       return 0;
     } else {
       LOG(ERROR) << "flush buffer fail:" << status.ToString();
@@ -93,13 +114,23 @@ int ResLevelDB::SetValue(const std::string& key, const std::string& value) {
 }
 
 std::string ResLevelDB::GetValue(const std::string& key) {
-  std::string value = "";
-  leveldb::Status status = db_->Get(leveldb::ReadOptions(), key, &value);
-  if (status.ok()) {
-    return value;
-  } else {
-    return "";
+  std::string value;
+  bool found_in_cache = false;
+
+  if (block_cache_) {
+    value = block_cache_->Get(key);
+    found_in_cache = !value.empty();
   }
+
+  if (!found_in_cache) {
+    leveldb::Status status = db_->Get(leveldb::ReadOptions(), key, &value);
+    if (!status.ok()) {
+      value.clear();  // Ensure value is empty if not found in DB
+    }
+  }
+
+  UpdateMetrics();
+  return value;
 }
 
 std::string ResLevelDB::GetAllValues(void) {
@@ -134,6 +165,19 @@ std::string ResLevelDB::GetRange(const std::string& min_key,
   return values;
 }
 
+bool ResLevelDB::UpdateMetrics() {
+  if (block_cache_ == nullptr) {
+    return false;
+  }
+  std::string stats;
+  std::string approximate_size;
+  db_->GetProperty("leveldb.stats", &stats);
+  db_->GetProperty("leveldb.approximate-memory-usage", &approximate_size);
+  global_stats_->SetStorageEngineMetrics(block_cache_->GetCacheHitRatio(),
+                                         stats, approximate_size);
+  return true;
+}
+
 bool ResLevelDB::Flush() {
   leveldb::Status status = db_->Write(leveldb::WriteOptions(), &batch_);
   if (status.ok()) {

chain/storage/leveldb.h

@@ -25,8 +25,10 @@
 
 #include "chain/storage/proto/leveldb_config.pb.h"
 #include "chain/storage/storage.h"
+#include "common/lru/lru_cache.h"
 #include "leveldb/db.h"
 #include "leveldb/write_batch.h"
+#include "platform/statistic/stats.h"
 
 namespace resdb {
 namespace storage {
@@ -65,6 +67,8 @@ class ResLevelDB : public Storage {
   std::vector<std::pair<std::string, int>> GetTopHistory(
       const std::string& key, int top_number) override;
 
+  bool UpdateMetrics();
+
   bool Flush() override;
 
  private:
@@ -75,6 +79,10 @@ class ResLevelDB : public Storage {
   ::leveldb::WriteBatch batch_;
   unsigned int write_buffer_size_ = 64 << 20;
   unsigned int write_batch_size_ = 1;
+
+ protected:
+  Stats* global_stats_ = nullptr;
+  std::unique_ptr<LRUCache<std::string, std::string>> block_cache_;
 };
 
 }  // namespace storage

chain/storage/leveldb_test.cpp

@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "chain/storage/leveldb.h"
+
+#include <glog/logging.h>
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include <filesystem>
+
+namespace resdb {
+namespace storage {
+namespace {
+
+enum class CacheConfig { DISABLED, ENABLED };
+
+class TestableResLevelDB : public ResLevelDB {
+ public:
+  using ResLevelDB::block_cache_;
+  using ResLevelDB::global_stats_;
+  using ResLevelDB::ResLevelDB;
+};
+
+class LevelDBTest : public ::testing::TestWithParam<CacheConfig> {
+ protected:
+  LevelDBTest() {
+    LevelDBInfo config;
+    if (GetParam() == CacheConfig::ENABLED) {
+      config.set_enable_block_cache(true);
+      config.set_block_cache_capacity(1000);
+    } else {
+      Reset();
+    }
+    storage = std::make_unique<TestableResLevelDB>(config);
+  }
+
+ protected:
+  std::unique_ptr<TestableResLevelDB> storage;
+  std::string path_ = "/tmp/leveldb_test";
+
+ private:
+  void Reset() { std::filesystem::remove_all(path_.c_str()); }
+};
+
+TEST_P(LevelDBTest, BlockCacheEnabled) {
+  if (GetParam() == CacheConfig::ENABLED) {
+    EXPECT_TRUE(storage->block_cache_ != nullptr);
+    EXPECT_TRUE(storage->block_cache_->GetCapacity() == 1000);
+  } else {
+    EXPECT_TRUE(storage->block_cache_ == nullptr);
+    EXPECT_FALSE(storage->UpdateMetrics());
+  }
+}
+
+TEST_P(LevelDBTest, AddValueAndCheckCache) {
+  if (GetParam() == CacheConfig::ENABLED) {
+    // Add a value
+    std::string key = "test_key";
+    std::string value = "test_value";
+    EXPECT_EQ(storage->SetValue(key, value), 0);
+
+    // Check if CacheHit is incremented in the Stats class
+    EXPECT_TRUE(storage->block_cache_->Get(key) == "test_value");
+    EXPECT_EQ(storage->block_cache_->GetCacheHits(), 1);
+  }
+}
+
+TEST_P(LevelDBTest, CacheEvictionPolicy) {
+  if (GetParam() == CacheConfig::ENABLED) {
+    // Insert 1000 values
+    for (int i = 1; i <= 1000; ++i) {
+      std::string key = "key_" + std::to_string(i);
+      std::string value = "value_" + std::to_string(i);
+      EXPECT_EQ(storage->SetValue(key, value), 0);
+    }
+
+    // Insert the 1001st value
+    std::string key_1001 = "key_1001";
+    std::string value_1001 = "value_1001";
+    EXPECT_EQ(storage->SetValue(key_1001, value_1001), 0);
+
+    // Check that the 1001st value is not present in the cache
+    EXPECT_TRUE(storage->GetValue("key_1") == "value_1");
+    EXPECT_EQ(storage->block_cache_->GetCacheMisses(), 1);
+
+    // Expect key_2 to be present in cache and hence a cache hit
+    EXPECT_TRUE(storage->GetValue("key_2") == "value_2");
+    EXPECT_EQ(storage->block_cache_->GetCacheHits(), 1);
+
+    EXPECT_TRUE(storage->UpdateMetrics());
+  }
+}
+
+INSTANTIATE_TEST_CASE_P(LevelDBTest, LevelDBTest,
+                        ::testing::Values(CacheConfig::ENABLED,
+                                          CacheConfig::DISABLED));
+
+}  // namespace
+}  // namespace storage
+}  // namespace resdb

chain/storage/proto/leveldb_config.proto

@@ -25,4 +25,6 @@ message LevelDBInfo {
   uint32 write_buffer_size_mb = 2;
   uint32 write_batch_size = 3;
   string path = 4;
+  optional bool enable_block_cache = 5;
+  optional uint32 block_cache_capacity = 6;
 }