Merge branch 'master' into scripts

Author: schedutron

Jenkinsfile

@@ -22,6 +22,7 @@ pipeline {
                         sh 'cd build && make benchmark -j4'
                         sh 'cd build && make install'
                         sh 'cd build && bash ../script/testing/psql/psql_test.sh'
+                        sh 'sudo apt-get -qq update && sudo apt-get -qq -y --no-install-recommends install wget default-jdk default-jre' // prerequisites for jdbc_validator
                         sh 'cd build && python ../script/validators/jdbc_validator.py'
                     }
                 }
@@ -48,6 +49,7 @@ pipeline {
                         // sh 'cd build && make benchmark -j4'
                         // sh 'cd build && make install'
                         // sh 'cd build && bash ../script/testing/psql/psql_test.sh'
+                        // sh 'sudo apt-get -qq update && sudo apt-get -qq -y --no-install-recommends install wget default-jdk default-jre' // prerequisites for jdbc_validator
                         // sh 'cd build && python ../script/validators/jdbc_validator.py'
                     }
                 }
@@ -64,6 +66,7 @@ pipeline {
                         sh 'cd build && make benchmark -j4'
                         sh 'cd build && make install'
                         sh 'cd build && bash ../script/testing/psql/psql_test.sh'
+                        sh 'sudo apt-get -qq update && sudo apt-get -qq -y --no-install-recommends install wget default-jdk default-jre' // prerequisites for jdbc_validator
                         sh 'cd build && python ../script/validators/jdbc_validator.py'
                     }
                 }
@@ -79,6 +82,7 @@ pipeline {
                 //         sh 'cd build && make benchmark -j4'
                 //         sh 'cd build && make install'
                 //         sh 'cd build && bash ../script/testing/psql/psql_test.sh'
+                //         sh 'sudo apt-get -qq update && sudo apt-get -qq -y --no-install-recommends install wget default-jdk default-jre' // prerequisites for jdbc_validator
                 //         sh 'cd build && python ../script/validators/jdbc_validator.py'
                 //     }
                 // }

script/git-hooks/pre-commit

@@ -18,8 +18,9 @@ if [ -n "$FILES" ]; then
     echo "***************************************"
     echo "******* Peloton Pre-Commit Hook *******"
     echo "***************************************"
-    echo "Use \"$FORMATTER_COMMAND -c -f\" to format all staged files."
-    echo "Or use \"$FORMATTER_COMMAND --no-verify\" to temporarily bypass the pre-commit hook."
+    echo "Use \"$FORMATTER_PATH -c -f\" to format all staged files."
+    echo "Or use \"git commit --no-verify\" to temporarily bypass the pre-commit hook."
+
     echo
     echo "Be aware that changed files have to be staged again!"
     echo "***************************************"

src/include/capnp/peloton_service.capnp

@@ -0,0 +1,20 @@
+@0xf3d342883f3f0344;
+
+struct CreateIndexRequest {
+  databaseName @0 :Text;
+  tableName @1 :Text;
+
+  keyAttrs @2 :List(Int32);
+  indexName @3 :Text;
+  uniqueKeys @4 :Bool;
+
+  indexKeys @5 :Int32;
+}
+
+struct CreateIndexResponse {
+
+}
+
+interface PelotonService {
+  createIndex @0 (request :CreateIndexRequest) -> (response :CreateIndexResponse);
+}

src/include/common/dedicated_thread_owner.h

@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+//                         Peloton
+//
+// dedicated_thread_owner.h
+//
+// Identification: src/include/common/dedicated_thread_owner.h
+//
+// Copyright (c) 2015-2018, Carnegie Mellon University Database Group
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+#include <memory>
+#include <thread>
+#include "common/dedicated_thread_task.h"
+
+namespace peloton {
+/**
+ * @brief DedicatedThreadOwner is the base class for all components that
+ * needs to manage long running threads inside the system (e.g. GC, thread pool)
+ *
+ * The interface exposes necessary behavior to @see DedicatedThreadRegistry, so
+ * that the system has a centralized record over all the threads currently
+ * running, and retains control over those threads for tuning purposes.
+ *
+ * TODO(tianyu): also add some statistics of thread utilization for tuning
+ */
+class DedicatedThreadOwner {
+ public:
+  /**
+   * @return the number of threads owned by this owner
+   */
+  size_t GetThreadCount() { return thread_count_; }
+
+  /**
+   * Notifies the owner that a new thread has been given to it
+   */
+  void NotifyNewThread() { thread_count_++; };
+
+  /**
+   * Notifies the owner that the thread running task will be terminated
+   * @param task the task to be terminated
+   */
+  void NotifyThreadRemoved(std::shared_ptr<DedicatedThreadTask> task) {
+    thread_count_--;
+    OnThreadRemoved(task);
+  }
+
+ protected:
+  /**
+   * Custom code to be run when removing a thread by each owner. It is expected
+   * that this function blocks until the thread can be dropped safely
+   *
+   * TODO(tianyu) turn into async if need be
+   */
+  virtual void OnThreadRemoved(std::shared_ptr<DedicatedThreadTask>) {}
+
+ private:
+  size_t thread_count_ = 0;
+};
+}  // namespace peloton

src/include/common/dedicated_thread_registry.h

@@ -0,0 +1,82 @@
+//===----------------------------------------------------------------------===//
+//
+//                         Peloton
+//
+// dedicated_thread_registry.h
+//
+// Identification: src/include/common/dedicated_thread_registry.h
+//
+// Copyright (c) 2015-2018, Carnegie Mellon University Database Group
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+#include <memory>
+#include <unordered_map>
+#include <vector>
+#include <thread>
+#include "common/macros.h"
+#include "common/dedicated_thread_task.h"
+#include "common/dedicated_thread_owner.h"
+
+namespace peloton {
+
+/**
+ * Singleton class responsible for maintaining and dispensing long running
+ * (dedicated) threads to other system components. The class also serves
+ * as a control panel for the brain component to be able to collect information
+ * on threads in the system and modify how threads are allocated.
+ */
+class DedicatedThreadRegistry {
+ public:
+  DedicatedThreadRegistry() = default;
+
+  ~DedicatedThreadRegistry() {
+    // Note that if registry is shutting down, it doesn't matter whether
+    // owners are notified as this class should have the same life cycle
+    // as the entire peloton process.
+
+    for (auto &entry : thread_owners_table_) {
+      for (auto &task : entry.second) {
+        task->Terminate();
+        threads_table_[task.get()].join();
+      }
+    }
+  }
+
+  // TODO(tianyu): Remove when we remove singletons
+  static DedicatedThreadRegistry &GetInstance()  {
+    static DedicatedThreadRegistry registry;
+    return registry;
+  }
+
+  /**
+   *
+   * Register a thread under requester to run the given task
+   *
+   * @param requester The owner to assign the new thread to
+   * @param args the arguments to pass to constructor of task
+   * @return the DedicatedThreadTask running on new thread
+   */
+  template <typename Task>
+  void RegisterDedicatedThread(DedicatedThreadOwner *requester,
+                      std::shared_ptr<Task> task) {
+    thread_owners_table_[requester].push_back(task);
+    requester->NotifyNewThread();
+    threads_table_.emplace(task.get(), std::thread([=] { task->RunTask(); }));
+  }
+
+  // TODO(tianyu): Add code for thread removal
+
+ private:
+  // Using raw pointer is okay since we never dereference said pointer,
+  // but only use it as a lookup key
+  std::unordered_map<DedicatedThreadTask *, std::thread> threads_table_;
+  // Using raw pointer here is also fine since the owner's life cycle is
+  // not controlled by the registry
+  std::unordered_map<DedicatedThreadOwner *,
+                     std::vector<std::shared_ptr<DedicatedThreadTask>>>
+      thread_owners_table_;
+};
+
+}  // namespace peloton

src/include/common/dedicated_thread_task.h

@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//
+//                         Peloton
+//
+// dedicated_thread_task.h
+//
+// Identification: src/include/common/dedicated_thread_task.h
+//
+// Copyright (c) 2015-2018, Carnegie Mellon University Database Group
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+namespace peloton {
+/**
+ * @brief Interface for a task to be run on a dedicated thread
+ *
+ * A dedicated thread is a long running thread that fulfills some system
+ * function running at all times. An example of this would be threads in
+ * the worker thread pool or the GC thread.
+ */
+class DedicatedThreadTask {
+ public:
+  /**
+   * Send a termination signal to the dedicated thread.
+   *
+   * The thread must then wrap up and exit from its Run function. The
+   * termination is guaranteed to be communicated to the owner
+   */
+  virtual void Terminate() = 0;
+
+  /**
+   * Executes the dedicated thread. It is assumed that the thread doesn't exit
+   * until terminate is explicitly called.
+   */
+  virtual void RunTask() = 0;
+};
+}  // namespace peloton

src/include/network/network_thread.h

@@ -0,0 +1,52 @@
+//===----------------------------------------------------------------------===//
+//
+//                         Peloton
+//
+// peloton_rpc_handler_task.h
+//
+// Identification: src/include/network/peloton_rpc_handler_task.h
+//
+// Copyright (c) 2015-2018, Carnegie Mellon University Database Group
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+#include "capnp/ez-rpc.h"
+#include "capnp/message.h"
+#include "common/dedicated_thread_task.h"
+#include "common/logger.h"
+#include "kj/debug.h"
+#include "peloton/capnp/peloton_service.capnp.h"
+
+namespace peloton {
+namespace network {
+class PelotonRpcServerImpl final : public PelotonService::Server {
+ protected:
+  kj::Promise<void> createIndex(CreateIndexContext) override {
+    // TODO(tianyu) Write actual index code
+    LOG_DEBUG("Received rpc to create index");
+    return kj::READY_NOW;
+  }
+};
+
+
+class PelotonRpcHandlerTask : public DedicatedThreadTask {
+ public:
+  explicit PelotonRpcHandlerTask(const char *address) : address_(address) {}
+
+  void Terminate() override {
+    // TODO(tianyu): Not implemented. See:
+    // https://groups.google.com/forum/#!topic/capnproto/bgxCdqGD6oE
+  }
+
+  void RunTask() override {
+    capnp::EzRpcServer server(kj::heap<PelotonRpcServerImpl>(), address_);
+    LOG_DEBUG("Server listening on %s", address_);
+    kj::NEVER_DONE.wait(server.getWaitScope());
+  }
+
+ private:
+  const char *address_;
+};
+}  // namespace network
+}  // namespace peloton