Merge pull request #1102 from pranavk/settings_gh

Allow setting number of GC threads from command line

Author: pervazea

src/common/init.cpp

@@ -56,6 +56,7 @@ void PelotonInit::Initialize() {
   concurrency::EpochManagerFactory::GetInstance().StartEpoch();
 
   // start GC.
+  gc::GCManagerFactory::Configure(settings::SettingsManager::GetInt(settings::SettingId::gc_num_threads));
   gc::GCManagerFactory::GetInstance().StartGC();
 
   // start index tuner

src/include/common/exception.h

@@ -76,6 +76,10 @@ class Exception : public std::runtime_error {
                          "\nMessage :: " + message;
   }
 
+  std::string GetMessage() {
+    return exception_message_;
+  }
+
   std::string ExceptionTypeToString(ExceptionType type) {
     switch (type) {
       case ExceptionType::INVALID:

src/include/settings/settings.h

@@ -21,18 +21,21 @@
 SETTING_int(port,
            "Peloton port (default: 15721)",
            15721,
+           1024, 65535,
            false, false)
 
 // Maximum number of connections
 SETTING_int(max_connections,
            "Maximum number of connections (default: 64)",
            64,
+           1, 512,
            true, true)
 
 SETTING_int(rpc_port,
             "Peloton rpc port (default: 15445)",
-             15445,
-             false, false)
+            15445,
+            1024, 65535,
+            false, false)
 
 // TODO(tianyu): Remove when we change to a different rpc framework
 // This is here only because capnp cannot exit gracefully and thus causes
@@ -82,24 +85,36 @@ SETTING_string(root_cert_file,
 SETTING_double(bnlj_buffer_size,
              "The default buffer size to use for blockwise nested loop joins (default: 1 MB)",
              1.0 * 1024.0 * 1024.0,
+             1.0 * 1024,
+             1.0 * 1024.0 * 1024.0 * 1024,
              true, true)
 
 // Size of the MonoQueue task queue
 SETTING_int(monoqueue_task_queue_size,
             "MonoQueue Task Queue Size (default: 32)",
-            32,
+            32, 
+            8, 128,
             false, false)
 
 // Size of the MonoQueue worker pool
 SETTING_int(monoqueue_worker_pool_size,
             "MonoQueue Worker Pool Size (default: 4)",
-            4,
+            4, 
+            1, 32,
             false, false)
 
 // Number of connection threads used by peloton
 SETTING_int(connection_thread_count,
             "Number of connection threads (default: std::hardware_concurrency())",
-            std::thread::hardware_concurrency(), false, false)
+            std::thread::hardware_concurrency(),
+            1, 64,
+            false, false)
+
+SETTING_int(gc_num_threads,
+            "The number of Garbage collection threads to run",
+            1,
+            1, 128,
+            true, true)
 
 //===----------------------------------------------------------------------===//
 // WRITE AHEAD LOG
@@ -127,6 +142,7 @@ SETTING_bool(display_settings,
 SETTING_int(stats_mode,
            "Enable statistics collection (default: 0)",
            static_cast<int>(peloton::StatsType::INVALID),
+           0, 16,
            true, true)
 
 //===----------------------------------------------------------------------===//
@@ -164,12 +180,14 @@ SETTING_string(peloton_address,
 SETTING_int(brain_task_queue_size,
             "Brain Task Queue Size (default: 32)",
             32,
+            1, 128,
             false, false)
 
 // Size of the brain worker pool
 SETTING_int(brain_worker_pool_size,
             "Brain Worker Pool Size (default: 1)",
             1,
+            1, 16,
             false, false)
 
 //===----------------------------------------------------------------------===//
@@ -199,7 +217,9 @@ SETTING_int(task_execution_timeout,
             "Maximum allowed length of time (in ms) for task "
                 "execution step of optimizer, "
                 "assuming one plan has been found (default 5000)",
-            5000, true, true)
+            5000,
+	    1000, 60000,
+	    true, true)
 
 //===----------------------------------------------------------------------===//
 // GENERAL

src/include/settings/settings_macro.h

@@ -33,16 +33,16 @@
   #ifdef SETTING_string
     #undef SETTING_string
   #endif
-  #define SETTING_int(name, description, default_value, is_mutable, is_persistent)     \
+  #define SETTING_int(name, description, default_value, min_value, max_value, is_mutable, is_persistent)        \
     DEFINE_int32(name, default_value, description);
 
-  #define SETTING_double(name, description, default_value, is_mutable, is_persistent)     \
+  #define SETTING_double(name, description, default_value, min_value, max_value, is_mutable, is_persistent)     \
     DEFINE_double(name, default_value, description);
 
-  #define SETTING_bool(name, description, default_value, is_mutable, is_persistent)    \
+  #define SETTING_bool(name, description, default_value, is_mutable, is_persistent)                             \
     DEFINE_bool(name, default_value, description);
 
-  #define SETTING_string(name, description, default_value, is_mutable, is_persistent)  \
+  #define SETTING_string(name, description, default_value, is_mutable, is_persistent)                           \
     DEFINE_string(name, default_value, description);
 #endif
 
@@ -59,16 +59,16 @@
   #ifdef SETTING_string
     #undef SETTING_string
   #endif
-  #define SETTING_int(name, description, default_value, is_mutable, is_persistent)     \
+  #define SETTING_int(name, description, default_value, min_value, max_value, is_mutable, is_persistent)        \
     DECLARE_int32(name);
 
-  #define SETTING_double(name, description, default_value, is_mutable, is_persistent)     \
+  #define SETTING_double(name, description, default_value, min_value, max_value, is_mutable, is_persistent)     \
     DECLARE_double(name);
 
-  #define SETTING_bool(name, description, default_value, is_mutable, is_persistent)    \
+  #define SETTING_bool(name, description, default_value, is_mutable, is_persistent)                             \
     DECLARE_bool(name);
 
-  #define SETTING_string(name, description, default_value, is_mutable, is_persistent)  \
+  #define SETTING_string(name, description, default_value, is_mutable, is_persistent)                           \
     DECLARE_string(name);
 #endif
 
@@ -85,32 +85,40 @@
   #ifdef SETTING_string
     #undef SETTING_string
   #endif
-  #define SETTING_int(name, description, default_value, is_mutable, is_persistent)     \
-      DefineSetting(                                                                   \
-        peloton::settings::SettingId::name,                                            \
-        #name, type::ValueFactory::GetIntegerValue(FLAGS_##name),                      \
-        description, type::ValueFactory::GetIntegerValue(default_value),               \
+  #define SETTING_int(name, description, default_value, min_value, max_value, is_mutable, is_persistent)      \
+      DefineSetting(                                                                                          \
+        peloton::settings::SettingId::name,                                                                   \
+        #name, type::ValueFactory::GetIntegerValue(FLAGS_##name),                                             \
+        description, type::ValueFactory::GetIntegerValue(default_value),                                      \
+        type::ValueFactory::GetIntegerValue(min_value),                                                       \
+        type::ValueFactory::GetIntegerValue(max_value),                                                       \
         is_mutable, is_persistent);
 
-  #define SETTING_double(name, description, default_value, is_mutable, is_persistent)  \
-      DefineSetting(                                                                   \
-        peloton::settings::SettingId::name,                                            \
-        #name, type::ValueFactory::GetDecimalValue(FLAGS_##name),                      \
-        description, type::ValueFactory::GetDecimalValue(default_value),               \
+  #define SETTING_double(name, description, default_value, min_value, max_value, is_mutable, is_persistent)   \
+      DefineSetting(                                                                                          \
+        peloton::settings::SettingId::name,                                                                   \
+        #name, type::ValueFactory::GetDecimalValue(FLAGS_##name),                                             \
+        description, type::ValueFactory::GetDecimalValue(default_value),                                      \
+        type::ValueFactory::GetDecimalValue(min_value),                                                       \
+        type::ValueFactory::GetDecimalValue(max_value),                                                       \
         is_mutable, is_persistent);
 
-  #define SETTING_bool(name, description, default_value, is_mutable, is_persistent)    \
-      DefineSetting(                                                                   \
-        peloton::settings::SettingId::name,                                            \
-        #name, type::ValueFactory::GetBooleanValue(FLAGS_##name),                      \
-        description, type::ValueFactory::GetBooleanValue(default_value),               \
+  #define SETTING_bool(name, description, default_value, is_mutable, is_persistent)                           \
+      DefineSetting(                                                                                          \
+        peloton::settings::SettingId::name,                                                                   \
+        #name, type::ValueFactory::GetBooleanValue(FLAGS_##name),                                             \
+        description, type::ValueFactory::GetBooleanValue(default_value),                                      \
+        type::ValueFactory::GetBooleanValue(default_value),                                                   \
+        type::ValueFactory::GetBooleanValue(default_value),                                                   \
         is_mutable, is_persistent);
 
-  #define SETTING_string(name, description, default_value, is_mutable, is_persistent)  \
-      DefineSetting(                                                                   \
-        peloton::settings::SettingId::name,                                            \
-        #name, type::ValueFactory::GetVarcharValue(FLAGS_##name),                      \
-        description, type::ValueFactory::GetVarcharValue(default_value),               \
+  #define SETTING_string(name, description, default_value, is_mutable, is_persistent)                         \
+      DefineSetting(                                                                                          \
+        peloton::settings::SettingId::name,                                                                   \
+        #name, type::ValueFactory::GetVarcharValue(FLAGS_##name),                                             \
+        description, type::ValueFactory::GetVarcharValue(default_value),                                      \
+        type::ValueFactory::GetVarcharValue(default_value),                                                   \
+        type::ValueFactory::GetVarcharValue(default_value),                                                   \
         is_mutable, is_persistent);
 #endif
 
@@ -127,15 +135,15 @@
   #ifdef SETTING_string
     #undef SETTING_string
   #endif
-  #define SETTING_int(name, description, default_value, is_mutable, is_persistent)     \
+  #define SETTING_int(name, description, default_value, min_value, max_value, is_mutable, is_persistent)        \
     name,
 
-  #define SETTING_double(name, description, default_value, is_mutable, is_persistent)     \
+  #define SETTING_double(name, description, default_value, min_value, max_value, is_mutable, is_persistent)     \
     name,
 
-  #define SETTING_bool(name, description, default_value, is_mutable, is_persistent)    \
+  #define SETTING_bool(name, description, default_value, is_mutable, is_persistent)                             \
     name,
 
-  #define SETTING_string(name, description, default_value, is_mutable, is_persistent)  \
+  #define SETTING_string(name, description, default_value, is_mutable, is_persistent)                           \
     name,
 #endif

src/include/settings/settings_manager.h

@@ -82,6 +82,8 @@ class SettingsManager : public Printable {
                      const type::Value &value,
                      const std::string &description,
                      const type::Value &default_value,
+                     const type::Value &min_value,
+                     const type::Value &max_value,
                      bool is_mutable, bool is_persistent);
 
   type::Value GetValue(SettingId id);

src/include/type/value.h

@@ -129,6 +129,10 @@ class Value : public Printable {
   inline CmpBool CompareGreaterThanEquals(const Value &o) const {
     return Type::GetInstance(type_id_)->CompareGreaterThanEquals(*this, o);
   }
+  inline bool CompareBetweenInclusive(const Value &a, const Value &b) const {
+    return Type::GetInstance(type_id_)->CompareGreaterThanEquals(*this, a) == CmpBool::CmpTrue &&
+           Type::GetInstance(type_id_)->CompareLessThanEquals(*this, b) == CmpBool::CmpTrue;
+  }
 
   // Other mathematical functions
   inline Value Add(const Value &o) const {

src/main/peloton/peloton.cpp

@@ -75,11 +75,16 @@ int main(int argc, char *argv[]) {
     ::google::HandleCommandLineHelpFlags();
   }
 
-  // Print settings
-  if (peloton::settings::SettingsManager::GetBool(
-          peloton::settings::SettingId::display_settings)) {
-    auto &settings = peloton::settings::SettingsManager::GetInstance();
-    settings.ShowInfo();
+  try {
+    // Print settings
+    if (peloton::settings::SettingsManager::GetBool(
+      peloton::settings::SettingId::display_settings)) {
+      auto &settings = peloton::settings::SettingsManager::GetInstance();
+      settings.ShowInfo();
+    }
+  } catch (peloton::SettingsException &exception) {
+    peloton::LOG_ERROR("Cannot load settings. Failed with %s\n", exception.GetMessage().c_str());
+    return 0; // TODO: Use an enum with exit error codes
   }
 
   if (peloton::settings::SettingsManager::GetBool(

src/settings/settings_manager.cpp

@@ -12,6 +12,7 @@
 
 #include <gflags/gflags.h>
 
+#include "type/type_id.h"
 #include "catalog/settings_catalog.h"
 #include "common/exception.h"
 #include "concurrency/transaction_manager_factory.h"
@@ -116,6 +117,8 @@ const std::string SettingsManager::GetInfo() const {
                          GetBool(SettingId::codegen) ? "enabled" : "disabled"));
   info.append(StringUtil::Format("%28s:   %-28i\n", "Optimization Timeout",
                                  GetInt(SettingId::task_execution_timeout)));
+  info.append(StringUtil::Format("%28s:   %-28i\n", "Number of GC threads", 
+                                 GetInt(SettingId::gc_num_threads)));
   return StringBoxUtil::Box(info);
 }
 
@@ -125,10 +128,25 @@ void SettingsManager::DefineSetting(SettingId id, const std::string &name,
                                     const type::Value &value,
                                     const std::string &description,
                                     const type::Value &default_value,
+                                    const type::Value &min_value,
+                                    const type::Value &max_value,
                                     bool is_mutable, bool is_persistent) {
   if (settings_.find(id) != settings_.end()) {
     throw SettingsException("settings " + name + " already exists");
   }
+
+  // Only below types support min-max bound checking
+  if (value.GetTypeId() == type::TypeId::INTEGER ||
+      value.GetTypeId() == type::TypeId::SMALLINT ||
+      value.GetTypeId() == type::TypeId::TINYINT ||
+      value.GetTypeId() == type::TypeId::DECIMAL) {
+    if (!value.CompareBetweenInclusive(min_value, max_value))
+      throw SettingsException("Value given for \"" + name +
+                              "\" is not in its min-max bounds (" +
+                              min_value.ToString() + "-" +
+                              max_value.ToString() + ")");
+  }
+
   settings_.emplace(id, Param(name, value, description, default_value,
                               is_mutable, is_persistent));
 }