Merge branch 'master' into join-reordering

Author: GustavoAngulo

Jenkinsfile

@@ -22,6 +22,7 @@ pipeline {
                         sh 'cd build && make benchmark -j4'
                         sh 'cd build && make install'
                         sh 'cd build && bash ../script/testing/psql/psql_test.sh'
+                        sh 'sudo apt-get -qq update && sudo apt-get -qq -y --no-install-recommends install wget default-jdk default-jre' // prerequisites for jdbc_validator
                         sh 'cd build && python ../script/validators/jdbc_validator.py'
                     }
                 }
@@ -48,6 +49,7 @@ pipeline {
                         // sh 'cd build && make benchmark -j4'
                         // sh 'cd build && make install'
                         // sh 'cd build && bash ../script/testing/psql/psql_test.sh'
+                        // sh 'sudo apt-get -qq update && sudo apt-get -qq -y --no-install-recommends install wget default-jdk default-jre' // prerequisites for jdbc_validator
                         // sh 'cd build && python ../script/validators/jdbc_validator.py'
                     }
                 }
@@ -64,6 +66,7 @@ pipeline {
                         sh 'cd build && make benchmark -j4'
                         sh 'cd build && make install'
                         sh 'cd build && bash ../script/testing/psql/psql_test.sh'
+                        sh 'sudo apt-get -qq update && sudo apt-get -qq -y --no-install-recommends install wget default-jdk default-jre' // prerequisites for jdbc_validator
                         sh 'cd build && python ../script/validators/jdbc_validator.py'
                     }
                 }
@@ -79,6 +82,7 @@ pipeline {
                 //         sh 'cd build && make benchmark -j4'
                 //         sh 'cd build && make install'
                 //         sh 'cd build && bash ../script/testing/psql/psql_test.sh'
+                //         sh 'sudo apt-get -qq update && sudo apt-get -qq -y --no-install-recommends install wget default-jdk default-jre' // prerequisites for jdbc_validator
                 //         sh 'cd build && python ../script/validators/jdbc_validator.py'
                 //     }
                 // }

script/git-hooks/pre-commit

@@ -19,7 +19,7 @@ if [ -n "$FILES" ]; then
     echo "******* Peloton Pre-Commit Hook *******"
     echo "***************************************"
     echo "Use \"$FORMATTER_PATH -c -f\" to format all staged files."
-    echo "Or use \"$FORMATTER_PATH --no-verify\" to temporarily bypass the pre-commit hook."
+    echo "Or use \"git commit --no-verify\" to temporarily bypass the pre-commit hook."
     echo
     echo "Be aware that changed files have to be staged again!"
     echo "***************************************"

src/executor/create_executor.cpp

@@ -203,6 +203,7 @@ bool CreateExecutor::CreateTable(const planner::CreatePlan &node) {
 
 bool CreateExecutor::CreateIndex(const planner::CreatePlan &node) {
   auto txn = context_->GetTransaction();
+  auto database_name = node.GetDatabaseName();
   std::string table_name = node.GetTableName();
   std::string index_name = node.GetIndexName();
   bool unique_flag = node.IsUnique();
@@ -211,7 +212,7 @@ bool CreateExecutor::CreateIndex(const planner::CreatePlan &node) {
   auto key_attrs = node.GetKeyAttrs();
 
   ResultType result = catalog::Catalog::GetInstance()->CreateIndex(
-      DEFAULT_DB_NAME, table_name, key_attrs, index_name, unique_flag,
+      database_name, table_name, key_attrs, index_name, unique_flag,
       index_type, txn);
   txn->SetResult(result);
 

src/executor/plan_executor.cpp

@@ -64,7 +64,7 @@ static void CompileAndExecutePlan(
   }
 
   auto on_query_result =
-      [&on_complete, &consumer](executor::ExecutionResult result) {
+    [&on_complete, &consumer, plan](executor::ExecutionResult result) {
         std::vector<ResultValue> values;
         for (const auto &tuple : consumer.GetOutputTuples()) {
           for (uint32_t i = 0; i < tuple.tuple_.size(); i++) {
@@ -74,6 +74,7 @@ static void CompileAndExecutePlan(
             values.push_back(std::move(str));
           }
         }
+        plan->ClearParameterValues();
         on_complete(result, std::move(values));
       };
 
@@ -102,6 +103,7 @@ static void InterpretPlan(
     result.m_result = ResultType::FAILURE;
     result.m_error_message = "Failed initialization of query execution tree";
     CleanExecutorTree(executor_tree.get());
+    plan->ClearParameterValues();
     on_complete(result, std::move(values));
     return;
   }
@@ -131,6 +133,7 @@ static void InterpretPlan(
   result.m_processed = executor_context->num_processed;
   result.m_result = ResultType::SUCCESS;
   CleanExecutorTree(executor_tree.get());
+  plan->ClearParameterValues();
   on_complete(result, std::move(values));
 }
 
@@ -174,7 +177,7 @@ void PlanExecutor::ExecutePlan(
  * @return number of executed tuples and logical_tile_list
  */
 int PlanExecutor::ExecutePlan(
-    const planner::AbstractPlan *plan, const std::vector<type::Value> &params,
+    planner::AbstractPlan *plan, const std::vector<type::Value> &params,
     std::vector<std::unique_ptr<executor::LogicalTile>> &logical_tile_list) {
   PL_ASSERT(plan != nullptr);
   LOG_TRACE("PlanExecutor Start with transaction");

src/include/capnp/peloton_service.capnp

@@ -0,0 +1,20 @@
+@0xf3d342883f3f0344;
+
+struct CreateIndexRequest {
+  databaseName @0 :Text;
+  tableName @1 :Text;
+
+  keyAttrs @2 :List(Int32);
+  indexName @3 :Text;
+  uniqueKeys @4 :Bool;
+
+  indexKeys @5 :Int32;
+}
+
+struct CreateIndexResponse {
+
+}
+
+interface PelotonService {
+  createIndex @0 (request :CreateIndexRequest) -> (response :CreateIndexResponse);
+}

src/include/common/dedicated_thread_owner.h

@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+//                         Peloton
+//
+// dedicated_thread_owner.h
+//
+// Identification: src/include/common/dedicated_thread_owner.h
+//
+// Copyright (c) 2015-2018, Carnegie Mellon University Database Group
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+#include <memory>
+#include <thread>
+#include "common/dedicated_thread_task.h"
+
+namespace peloton {
+/**
+ * @brief DedicatedThreadOwner is the base class for all components that
+ * needs to manage long running threads inside the system (e.g. GC, thread pool)
+ *
+ * The interface exposes necessary behavior to @see DedicatedThreadRegistry, so
+ * that the system has a centralized record over all the threads currently
+ * running, and retains control over those threads for tuning purposes.
+ *
+ * TODO(tianyu): also add some statistics of thread utilization for tuning
+ */
+class DedicatedThreadOwner {
+ public:
+  /**
+   * @return the number of threads owned by this owner
+   */
+  size_t GetThreadCount() { return thread_count_; }
+
+  /**
+   * Notifies the owner that a new thread has been given to it
+   */
+  void NotifyNewThread() { thread_count_++; };
+
+  /**
+   * Notifies the owner that the thread running task will be terminated
+   * @param task the task to be terminated
+   */
+  void NotifyThreadRemoved(std::shared_ptr<DedicatedThreadTask> task) {
+    thread_count_--;
+    OnThreadRemoved(task);
+  }
+
+ protected:
+  /**
+   * Custom code to be run when removing a thread by each owner. It is expected
+   * that this function blocks until the thread can be dropped safely
+   *
+   * TODO(tianyu) turn into async if need be
+   */
+  virtual void OnThreadRemoved(std::shared_ptr<DedicatedThreadTask>) {}
+
+ private:
+  size_t thread_count_ = 0;
+};
+}  // namespace peloton

src/include/common/dedicated_thread_registry.h

@@ -0,0 +1,82 @@
+//===----------------------------------------------------------------------===//
+//
+//                         Peloton
+//
+// dedicated_thread_registry.h
+//
+// Identification: src/include/common/dedicated_thread_registry.h
+//
+// Copyright (c) 2015-2018, Carnegie Mellon University Database Group
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+#include <memory>
+#include <unordered_map>
+#include <vector>
+#include <thread>
+#include "common/macros.h"
+#include "common/dedicated_thread_task.h"
+#include "common/dedicated_thread_owner.h"
+
+namespace peloton {
+
+/**
+ * Singleton class responsible for maintaining and dispensing long running
+ * (dedicated) threads to other system components. The class also serves
+ * as a control panel for the brain component to be able to collect information
+ * on threads in the system and modify how threads are allocated.
+ */
+class DedicatedThreadRegistry {
+ public:
+  DedicatedThreadRegistry() = default;
+
+  ~DedicatedThreadRegistry() {
+    // Note that if registry is shutting down, it doesn't matter whether
+    // owners are notified as this class should have the same life cycle
+    // as the entire peloton process.
+
+    for (auto &entry : thread_owners_table_) {
+      for (auto &task : entry.second) {
+        task->Terminate();
+        threads_table_[task.get()].join();
+      }
+    }
+  }
+
+  // TODO(tianyu): Remove when we remove singletons
+  static DedicatedThreadRegistry &GetInstance()  {
+    static DedicatedThreadRegistry registry;
+    return registry;
+  }
+
+  /**
+   *
+   * Register a thread under requester to run the given task
+   *
+   * @param requester The owner to assign the new thread to
+   * @param args the arguments to pass to constructor of task
+   * @return the DedicatedThreadTask running on new thread
+   */
+  template <typename Task>
+  void RegisterDedicatedThread(DedicatedThreadOwner *requester,
+                      std::shared_ptr<Task> task) {
+    thread_owners_table_[requester].push_back(task);
+    requester->NotifyNewThread();
+    threads_table_.emplace(task.get(), std::thread([=] { task->RunTask(); }));
+  }
+
+  // TODO(tianyu): Add code for thread removal
+
+ private:
+  // Using raw pointer is okay since we never dereference said pointer,
+  // but only use it as a lookup key
+  std::unordered_map<DedicatedThreadTask *, std::thread> threads_table_;
+  // Using raw pointer here is also fine since the owner's life cycle is
+  // not controlled by the registry
+  std::unordered_map<DedicatedThreadOwner *,
+                     std::vector<std::shared_ptr<DedicatedThreadTask>>>
+      thread_owners_table_;
+};
+
+}  // namespace peloton

src/include/common/dedicated_thread_task.h

@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//
+//                         Peloton
+//
+// dedicated_thread_task.h
+//
+// Identification: src/include/common/dedicated_thread_task.h
+//
+// Copyright (c) 2015-2018, Carnegie Mellon University Database Group
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+namespace peloton {
+/**
+ * @brief Interface for a task to be run on a dedicated thread
+ *
+ * A dedicated thread is a long running thread that fulfills some system
+ * function running at all times. An example of this would be threads in
+ * the worker thread pool or the GC thread.
+ */
+class DedicatedThreadTask {
+ public:
+  /**
+   * Send a termination signal to the dedicated thread.
+   *
+   * The thread must then wrap up and exit from its Run function. The
+   * termination is guaranteed to be communicated to the owner
+   */
+  virtual void Terminate() = 0;
+
+  /**
+   * Executes the dedicated thread. It is assumed that the thread doesn't exit
+   * until terminate is explicitly called.
+   */
+  virtual void RunTask() = 0;
+};
+}  // namespace peloton

src/include/executor/plan_executor.h

@@ -70,7 +70,7 @@ class PlanExecutor {
    */
   // FIXME This should be removed when PelotonService is removed/rewritten
   static int ExecutePlan(
-      const planner::AbstractPlan *plan, const std::vector<type::Value> &params,
+      planner::AbstractPlan *plan, const std::vector<type::Value> &params,
       std::vector<std::unique_ptr<executor::LogicalTile>> &logical_tile_list);
 };