Self-Driving Index/Knob Actions (#1426)

Author: linmagit

CMakeLists.txt

@@ -152,7 +152,7 @@ print_sys_info("OS_NAME;OS_RELEASE;OS_VERSION;OS_PLATFORM")
 # CMake options, specify with -DNOISEPAGE_{option}=On.
 #   NOISEPAGE_BUILD_BENCHMARKS              : Enable building benchmarks as part of the ALL target. Default ON.
 #   NOISEPAGE_BUILD_TESTS                   : Enable building tests as part of the ALL (but the Self-Driving test) target. Default ON.
-#   NOISEPAGE_BUILD_SELF_DRIVING_TESTS      : Enable building self-driving end-to-end tests. Default OFF
+#   NOISEPAGE_BUILD_SELF_DRIVING_E2E_TESTS  : Enable building self-driving end-to-end tests. Default OFF
 #   NOISEPAGE_GENERATE_COVERAGE             : Enable C++ code coverage. Default OFF.
 #   NOISEPAGE_UNITTEST_OUTPUT_ON_FAILURE    : Enable verbose unittest failures. Default OFF. Can be very verbose.
 #   NOISEPAGE_UNITY_BUILD                   : Enable unity (aka jumbo) builds. Default OFF.
@@ -185,8 +185,8 @@ option(NOISEPAGE_BUILD_TESTS
         "Enable building tests as part of the ALL target."
         ON)
 
-option(NOISEPAGE_BUILD_SELF_DRIVING_TESTS
-        "Enable building self-driving tests as part of the ALL target."
+option(NOISEPAGE_BUILD_SELF_DRIVING_E2E_TESTS
+        "Enable building self-driving end-to-end tests as part of the ALL target."
         OFF)
 
 option(NOISEPAGE_GENERATE_COVERAGE
@@ -815,12 +815,12 @@ add_custom_target(jumbotests
         -T Test                             # Run tests and log it to Testing/*/Test.xml.
         --timeout 3000                      # 3000 second timeout per test.
         USES_TERMINAL)
-add_custom_target(self_driving_test         # For now, this target is specifically used for self-driving pipeline
+add_custom_target(self_driving_e2e_test         # For now, this target is specifically used for self-driving pipeline
         ctest
         --resource-spec-file ${BUILD_SUPPORT_DATA_DIR}/ctest_resource_specs.json    # For controlling conflicting tests.
         ${UNITTEST_OUTPUT_ON_FAILURE}       # Whether to print output when a test fails.
         -j ${NOISEPAGE_TEST_PARALLELISM}    # Maximum number of parallel jobs.
-        -L self_driving_test                # Run all tests that have a label like this. See footgun warning above.
+        -L self_driving_e2e_test                # Run all tests that have a label like this. See footgun warning above.
         --no-compress-output                # Output verbosely so that it can be logged.
         -T Test                             # Run tests and log it to Testing/*/Test.xml.
         --timeout 3000                      # 3000 second timeout per test.
@@ -907,8 +907,9 @@ function(add_noisepage_test
     add_dependencies(${TEST_LABEL} ${TEST_NAME})
 endfunction()
 
-# NOTE:Self-driving End-To-End tests are not included below because the test require dependencies to other part of the system, thus could not be run as standalone tests.
-# They will be added to a separate test target: self_driving_test. See NOISEPAGE_SELF_DRIVING_TEST_SOURCES for details
+# NOTE:Self-driving End-To-End tests are not included below because the test require dependencies to other part of
+# the system, thus could not be run as standalone tests.  They will be added to a separate test target:
+# self_driving_e2e_test. See NOISEPAGE_SELF_DRIVING_E2E_TEST_SOURCES for details.
 file(GLOB_RECURSE NOISEPAGE_TEST_SOURCES
         "test/binder/*.cpp"
         "test/catalog/*.cpp"
@@ -921,6 +922,7 @@ file(GLOB_RECURSE NOISEPAGE_TEST_SOURCES
         "test/optimizer/*.cpp"
         "test/parser/*.cpp"
         "test/planner/*.cpp"
+        "test/self_driving/*.cpp"
         "test/settings/*.cpp"
         "test/storage/*.cpp"
         "test/traffic_cop/*.cpp"
@@ -945,22 +947,22 @@ foreach (NOISEPAGE_TEST_CPP ${NOISEPAGE_TEST_SOURCES})
 endforeach ()
 
 
-file(GLOB_RECURSE NOISEPAGE_SELF_DRIVING_TEST_SOURCES
-        "test/self_driving/*.cpp"
+file(GLOB_RECURSE NOISEPAGE_SELF_DRIVING_E2E_TEST_SOURCES
+        "test/self_driving_e2e/*.cpp"
         )
 
-foreach (NOISEPAGE_TEST_CPP ${NOISEPAGE_SELF_DRIVING_TEST_SOURCES})
+foreach (NOISEPAGE_TEST_CPP ${NOISEPAGE_SELF_DRIVING_E2E_TEST_SOURCES})
     file(RELATIVE_PATH NOISEPAGE_TEST_CPP_REL "${PROJECT_SOURCE_DIR}/test" ${NOISEPAGE_TEST_CPP})
     get_filename_component(NOISEPAGE_TEST_DIR ${NOISEPAGE_TEST_CPP_REL} DIRECTORY)
     get_filename_component(NOISEPAGE_TEST ${NOISEPAGE_TEST_CPP} NAME_WE)
 
-    if (NOT ${NOISEPAGE_BUILD_SELF_DRIVING_TESTS})
+    if (NOT ${NOISEPAGE_BUILD_SELF_DRIVING_E2E_TESTS})
         set(EXCLUDE_OR_NOT "EXCLUDE_ALL")
     else ()
         set(EXCLUDE_OR_NOT "NO_EXCLUDE")
     endif ()
 
-    add_noisepage_test(${NOISEPAGE_TEST} ${NOISEPAGE_TEST_CPP} self_driving_test ${EXCLUDE_OR_NOT} NO_UNITY)
+    add_noisepage_test(${NOISEPAGE_TEST} ${NOISEPAGE_TEST_CPP} self_driving_e2e_test ${EXCLUDE_OR_NOT} NO_UNITY)
 endforeach ()
 
 
@@ -1024,6 +1026,7 @@ add_jumbotest("test/network" "network_test;")
 add_jumbotest("test/optimizer" "hyperloglog_test;")
 add_jumbotest("test/parser" "")
 add_jumbotest("test/planner" "")
+add_jumbotest("test/self_driving" "")
 add_jumbotest("test/settings" "")
 add_jumbotest("test/storage" "block_access_controller_test;block_compactor_test;bwtree_test;bwtree_index_test;data_table_test;data_table_concurrent_test;hash_index_test;large_garbage_collector_test;log_test;tuple_access_strategy_test;")
 add_jumbotest("test/traffic_cop" "traffic_cop_test;")

Jenkinsfile

@@ -495,8 +495,8 @@ pipeline {
                         sh script: '''
                         cd build
                         export BUILD_ABS_PATH=`pwd`
-                        timeout 10m ninja self_driving_test
-                        ''', label: 'Running self-driving test'
+                        timeout 10m ninja self_driving_e2e_test
+                        ''', label: 'Running self-driving end-to-end test'
 
                         sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
                     }

script/model/training_util/global_data_constructing_util.py

@@ -32,7 +32,6 @@ def get_data(input_path, mini_model_map, model_results_path, warmup_period, use_
     :return: (GlobalResourceData list, GlobalImpactData list)
     """
     cache_file = input_path + '/global_model_data.pickle'
-    headers_file = input_path + '/global_model_headers.pickle'
     if os.path.exists(cache_file):
         with open(cache_file, 'rb') as pickle_file:
             resource_data_list, impact_data_list, data_info.RAW_FEATURES_CSV_INDEX, data_info.RAW_TARGET_CSV_INDEX, data_info.INPUT_CSV_INDEX, data_info.TARGET_CSV_INDEX = pickle.load(pickle_file)

src/include/optimizer/index_util.h

@@ -74,9 +74,11 @@ class IndexUtil {
    * @param allow_cves Allow CVEs
    * @param scan_type IndexScanType to utilize
    * @param bounds Relevant bounds for the index scan
-   * @returns Whether index can be used
+   * @returns The first element represents whether the index can be used, the second element represents whether
+   * all the predicates' indexable columns that we suport are covered by the given index (schema). Note that a "true"
+   * value for the second element does not guarantee an optimal index.
    */
-  static bool SatisfiesPredicateWithIndex(
+  static std::pair<bool, bool> SatisfiesPredicateWithIndex(
       catalog::CatalogAccessor *accessor, catalog::table_oid_t tbl_oid, const std::string &tbl_alias,
       catalog::index_oid_t index_oid, const std::vector<AnnotatedExpression> &predicates, bool allow_cves,
       planner::IndexScanType *scan_type,
@@ -94,9 +96,10 @@ class IndexUtil {
    * @param allow_cves Allow ColumnValueExpressions
    * @param idx_scan_type IndexScanType to utilize
    * @param bounds Relevant bounds for the index scan
-   * @returns Whether predicate can be utilized
+   * @returns The first element represents whether predicate can be utilized, the second element represents whether
+   * all the predicates' indexable columns that we suport are covered by the given index (schema)
    */
-  static bool CheckPredicates(
+  static std::pair<bool, bool> CheckPredicates(
       const catalog::IndexSchema &schema, catalog::table_oid_t tbl_oid, const std::string &tbl_alias,
       const std::unordered_map<catalog::col_oid_t, catalog::indexkeycol_oid_t> &lookup,
       const std::unordered_set<catalog::col_oid_t> &mapped_cols, const std::vector<AnnotatedExpression> &predicates,

src/include/optimizer/physical_operators.h

@@ -143,12 +143,14 @@ class IndexScan : public OperatorNodeContents<IndexScan> {
    * @param is_for_update whether the scan is used for update
    * @param scan_type IndexScanType
    * @param bounds Bounds for IndexScan
+   * @param cover_all_columns whether the index covers all indexable columns (that we support) in the predicates
    * @return an IndexScan operator
    */
   static Operator Make(catalog::db_oid_t database_oid, catalog::table_oid_t tbl_oid, catalog::index_oid_t index_oid,
                        std::vector<AnnotatedExpression> &&predicates, bool is_for_update,
                        planner::IndexScanType scan_type,
-                       std::unordered_map<catalog::indexkeycol_oid_t, std::vector<planner::IndexExpression>> bounds);
+                       std::unordered_map<catalog::indexkeycol_oid_t, std::vector<planner::IndexExpression>> bounds,
+                       bool cover_all_columns);
 
   /**
    * Copy
@@ -197,6 +199,11 @@ class IndexScan : public OperatorNodeContents<IndexScan> {
     return bounds_;
   }
 
+  /**
+   * @return whether the index covers all predicate columns
+   */
+  bool GetCoverAllColumns() const { return cover_all_columns_; }
+
  private:
   /**
    * OID of the database
@@ -232,6 +239,12 @@ class IndexScan : public OperatorNodeContents<IndexScan> {
    * Bounds
    */
   std::unordered_map<catalog::indexkeycol_oid_t, std::vector<planner::IndexExpression>> bounds_;
+
+  /**
+   *
+   * The index covers all indexable columns in the predicates
+   */
+  bool cover_all_columns_;
 };
 
 /**

src/include/planner/plannodes/index_scan_plan_node.h

@@ -110,6 +110,15 @@ class IndexScanPlanNode : public AbstractScanPlanNode {
       return *this;
     }
 
+    /**
+     * @param cover_all_columns whether the index covers all predicate columns
+     * @return builder object
+     */
+    Builder &SetCoverAllColumns(bool cover_all_columns) {
+      cover_all_columns_ = cover_all_columns;
+      return *this;
+    }
+
     /**
      * Build the Index scan plan node
      * @return plan node
@@ -125,6 +134,7 @@ class IndexScanPlanNode : public AbstractScanPlanNode {
     std::unordered_map<catalog::indexkeycol_oid_t, IndexExpression> lo_index_cols_{};
     std::unordered_map<catalog::indexkeycol_oid_t, IndexExpression> hi_index_cols_{};
     uint64_t index_size_{0};
+    bool cover_all_columns_{false};
   };
 
  private:
@@ -141,6 +151,7 @@ class IndexScanPlanNode : public AbstractScanPlanNode {
    * @param lo_index_cols lower bound of the scan (or exact key when scan type = Exact).
    * @param hi_index_cols upper bound of the scan
    * @param index_size number of tuples in index
+   * @param cover_all_columns whether the index covers all predicate columns
    * @param plan_node_id Plan node id
    */
   IndexScanPlanNode(std::vector<std::unique_ptr<AbstractPlanNode>> &&children,
@@ -151,7 +162,7 @@ class IndexScanPlanNode : public AbstractScanPlanNode {
                     std::unordered_map<catalog::indexkeycol_oid_t, IndexExpression> &&lo_index_cols,
                     std::unordered_map<catalog::indexkeycol_oid_t, IndexExpression> &&hi_index_cols,
                     uint32_t scan_limit, bool scan_has_limit, uint32_t scan_offset, bool scan_has_offset,
-                    uint64_t index_size, uint64_t table_num_tuple, plan_node_id_t plan_node_id);
+                    uint64_t index_size, uint64_t table_num_tuple, bool cover_all_columns, plan_node_id_t plan_node_id);
 
  public:
   /**
@@ -217,6 +228,11 @@ class IndexScanPlanNode : public AbstractScanPlanNode {
    */
   PlanNodeType GetPlanNodeType() const override { return PlanNodeType::INDEXSCAN; }
 
+  /**
+   * @return whether the index covers all predicate columns
+   */
+  bool GetCoverAllColumns() const { return cover_all_columns_; }
+
   /**
    * @return the hashed value of this plan node
    */
@@ -237,6 +253,7 @@ class IndexScanPlanNode : public AbstractScanPlanNode {
   std::unordered_map<catalog::indexkeycol_oid_t, IndexExpression> hi_index_cols_{};
   uint64_t table_num_tuple_;
   uint64_t index_size_;
+  bool cover_all_columns_;
 };
 
 DEFINE_JSON_HEADER_DECLARATIONS(IndexScanPlanNode);

src/include/self_driving/pilot/action/abstract_action.h

@@ -0,0 +1,88 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "common/resource_tracker.h"
+#include "self_driving/pilot/action/action_defs.h"
+
+namespace noisepage::selfdriving::pilot {
+
+/**
+ * The abstract class for self-driving actions
+ */
+class AbstractAction {
+ public:
+  /**
+   * Constructor for the base AbstractAction.
+   * @param family The family that this action belongs to
+   */
+  explicit AbstractAction(ActionType family) : action_family_(family), id_(action_id_counter++) {}
+
+  virtual ~AbstractAction() = default;
+
+  /**
+   * Set the estimated runtime metrics for this action
+   * @param estimated_metrics The metrics to set to
+   */
+  void SetEstimatedMetrics(const common::ResourceTracker::Metrics &estimated_metrics) {
+    estimated_metrics_ = estimated_metrics;
+  }
+
+  /** @return The estimated runtime metrics for this action */
+  const common::ResourceTracker::Metrics &GetActualMetrics() { return estimated_metrics_; }
+
+  /** @return This action's ID */
+  action_id_t GetActionID() const { return id_; }
+
+  /** @return This action's family */
+  ActionType GetActionFamily() const { return action_family_; }
+
+  /**
+   * Add an equivalent action
+   * @param id Action ID
+   */
+  void AddEquivalentAction(action_id_t id) { equivalent_action_ids_.emplace_back(id); }
+
+  /**
+   * Get the equivalent action ids
+   * @return Action ID vector
+   */
+  const std::vector<action_id_t> &GetEquivalentActions() const { return equivalent_action_ids_; }
+
+  /**
+   * Add a reverse action
+   * @param id Action ID
+   */
+  void AddReverseAction(action_id_t id) { reverse_action_ids_.emplace_back(id); }
+
+  /**
+   * Get the reverse action ids
+   * @return Action ID vector
+   */
+  const std::vector<action_id_t> &GetReverseActions() const { return reverse_action_ids_; }
+
+  /**
+   * Get the SQL command to apply the action
+   * @return Action SQL command
+   */
+  virtual const std::string &GetSQLCommand() { return sql_command_; }
+
+ protected:
+  std::string sql_command_;  ///< The SQL commaned used to apply the action
+
+ private:
+  static action_id_t action_id_counter;
+
+  common::ResourceTracker::Metrics estimated_metrics_{};
+
+  ActionType action_family_;
+
+  /** ID is unique for an action among on planning process (one MCTS) */
+  action_id_t id_;
+
+  std::vector<action_id_t> equivalent_action_ids_;
+  std::vector<action_id_t> reverse_action_ids_;
+};
+
+}  // namespace noisepage::selfdriving::pilot

src/include/self_driving/pilot/action/action_defs.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "common/strong_typedef.h"
+
+namespace noisepage::selfdriving::pilot {
+
+/**
+ * typedef for action ID
+ */
+STRONG_TYPEDEF_HEADER(action_id_t, int32_t);
+
+/**
+ * Metric types
+ */
+enum class ActionType : uint8_t {
+  CREATE_INDEX,
+  DROP_INDEX,
+  CHANGE_KNOB,
+};
+
+}  // namespace noisepage::selfdriving::pilot

src/include/self_driving/pilot/action/change_knob_action.h

@@ -0,0 +1,45 @@
+#pragma once
+
+#include <string>
+#include <utility>
+
+#include "self_driving/pilot/action/abstract_action.h"
+#include "settings/settings_param.h"
+
+namespace noisepage::settings {
+class SettingsManager;
+}
+
+namespace noisepage::selfdriving::pilot {
+
+/**
+ * Represent a change knob self-driving action
+ */
+template <class T>
+class ChangeKnobAction : public AbstractAction {
+ public:
+  /**
+   * Construct ChangeKnobAction
+   * @param param Which knob param
+   * @param param_name Name of the param
+   * @param change_value The value to change that knob with
+   * @param settings_manager SettingsManager (used to find our the current knob value before applying the change_value)
+   */
+  ChangeKnobAction(settings::Param param, std::string param_name, T change_value,
+                   common::ManagedPointer<settings::SettingsManager> settings_manager)
+      : AbstractAction(ActionType::CHANGE_KNOB),
+        param_(param),
+        param_name_(std::move(param_name)),
+        change_value_(change_value),
+        settings_manager_(settings_manager) {}
+
+  const std::string &GetSQLCommand() override;
+
+ private:
+  settings::Param param_;
+  std::string param_name_;
+  T change_value_;
+  common::ManagedPointer<settings::SettingsManager> settings_manager_;
+};
+
+}  // namespace noisepage::selfdriving::pilot