Add more fields to Query Plan capnp and other improvements. (#4633)

This PR includes some fixes to the cap'n proto design of Query Plan :
- Add as many fields as possible to the cap'n proto model, so that we
don't rely on their availability on the client
- Do not display `VFS.Backend` field in JSON for remote arrays
- Remove `query_plan` toggle from `query_from_capnp`, and replace with
generic `allocate_buffers` flag that transfers the responsibility of
deciding if query buffers should be allocated to the caller and removes
magic `true`/`false` values from the code.

(+ small fixes to array directory serialization that I stumbled upon)

---
TYPE: NO_HISTORY
DESC: Add more fields to Query Plan capnp and other improvements

Author: ypatia

tiledb/sm/filesystem/uri.cc

@@ -327,13 +327,7 @@ std::string URI::to_path(const std::string& uri) {
 
 std::string URI::backend_name() const {
   if (is_tiledb(uri_)) {
-    std::string array_ns, array_uri;
-    throw_if_not_ok(URI(uri_).get_rest_components(&array_ns, &array_uri));
-    auto prefix = array_uri.substr(0, array_uri.find_first_of(':'));
-    if (prefix == array_uri) {  // no `:` separator found in URI
-      prefix = "Unknown";
-    }
-    return prefix;
+    return "";
   } else {
     return uri_.substr(0, uri_.find_first_of(':'));
   }

tiledb/sm/query_plan/query_plan.cc

@@ -48,18 +48,14 @@ namespace sm {
 /* ********************************* */
 /*     CONSTRUCTORS & DESTRUCTORS    */
 /* ********************************* */
-QueryPlan::QueryPlan(Query& query, std::string strategy) {
+QueryPlan::QueryPlan(Query& query) {
   array_uri_ = query.array()->array_uri().to_string();
   vfs_backend_ = URI(array_uri_).backend_name();
   query_layout_ = query.layout();
 
-  if (strategy.empty()) {
-    // This most probably ends up creating the strategy on the query
-    auto strategy_ptr = query.strategy();
-    strategy_name_ = strategy_ptr->name();
-  } else {
-    strategy_name_ = strategy;
-  }
+  // This most probably ends up creating the strategy on the query
+  auto strategy_ptr = query.strategy();
+  strategy_name_ = strategy_ptr->name();
 
   array_type_ = query.array()->array_schema_latest().array_type();
 
@@ -78,19 +74,47 @@ QueryPlan::QueryPlan(Query& query, std::string strategy) {
   std::sort(dimensions_.begin(), dimensions_.end());
 }
 
+QueryPlan::QueryPlan(
+    Query& query,
+    Layout layout,
+    const std::string& strategy_name,
+    ArrayType array_type,
+    const std::vector<std::string>& attributes,
+    const std::vector<std::string>& dimensions)
+    : array_uri_{query.array()->array_uri().to_string()}
+    , vfs_backend_{URI(array_uri_).backend_name()}
+    , query_layout_{layout}
+    , strategy_name_{strategy_name}
+    , array_type_{array_type}
+    , attributes_{attributes}
+    , dimensions_{dimensions} {
+}
+
 /* ********************************* */
 /*                API                */
 /* ********************************* */
 std::string QueryPlan::dump_json(uint32_t indent) {
-  nlohmann::json rv = {
-      {"TileDB Query Plan",
-       {{"Array.URI", array_uri_},
-        {"Array.Type", array_type_str(array_type_)},
-        {"VFS.Backend", vfs_backend_},
-        {"Query.Layout", layout_str(query_layout_)},
-        {"Query.Strategy.Name", strategy_name_},
-        {"Query.Attributes", attributes_},
-        {"Query.Dimensions", dimensions_}}}};
+  nlohmann::json rv;
+  if (!URI(array_uri_).is_tiledb()) {
+    rv = {
+        {"TileDB Query Plan",
+         {{"Array.URI", array_uri_},
+          {"Array.Type", array_type_str(array_type_)},
+          {"VFS.Backend", vfs_backend_},
+          {"Query.Layout", layout_str(query_layout_)},
+          {"Query.Strategy.Name", strategy_name_},
+          {"Query.Attributes", attributes_},
+          {"Query.Dimensions", dimensions_}}}};
+  } else {
+    rv = {
+        {"TileDB Query Plan",
+         {{"Array.URI", array_uri_},
+          {"Array.Type", array_type_str(array_type_)},
+          {"Query.Layout", layout_str(query_layout_)},
+          {"Query.Strategy.Name", strategy_name_},
+          {"Query.Attributes", attributes_},
+          {"Query.Dimensions", dimensions_}}}};
+  }
 
   return rv.dump(indent);
 }

tiledb/sm/query_plan/query_plan.h

@@ -65,9 +65,26 @@ class QueryPlan {
    * Constructor
    *
    * @param query A query object for which we want to calculate the plan
-   * @param strategy The strategy of the query, required only for remote queries
    */
-  QueryPlan(Query& query, const std::string strategy = "");
+  QueryPlan(Query& query);
+
+  /**
+   * Constructor
+   *
+   * @param query A query object for which we want to calculate the plan.
+   * @param layout The query layout.
+   * @param strategy_name The name of the strategy the query uses.
+   * @param array_type The type of the array.
+   * @param attributes The attribute names set in the query.
+   * @param dimensions The dimension names set in the query.
+   */
+  QueryPlan(
+      Query& query,
+      Layout layout,
+      const std::string& strategy_name,
+      ArrayType array_type,
+      const std::vector<std::string>& attributes,
+      const std::vector<std::string>& dimensions);
 
   /* ****************************** */
   /*              API               */
@@ -83,12 +100,40 @@ class QueryPlan {
   std::string dump_json(uint32_t indent = 4);
 
   /*
-   * Get the strategy name stored in the query plan
+   * Get the query layout stored in the query plan.
+   */
+  inline Layout query_layout() const {
+    return query_layout_;
+  }
+
+  /*
+   * Get the strategy name stored in the query plan.
    */
   inline std::string strategy() const {
     return strategy_name_;
   }
 
+  /*
+   * Get the array type stored in the query plan.
+   */
+  inline ArrayType array_type() const {
+    return array_type_;
+  }
+
+  /*
+   * Get the attribute names stored in the query plan.
+   */
+  inline std::vector<std::string> attributes() const {
+    return attributes_;
+  }
+
+  /*
+   * Get the dimension names stored in the query plan.
+   */
+  inline std::vector<std::string> dimensions() const {
+    return dimensions_;
+  }
+
  private:
   /* ****************************** */
   /*       PRIVATE ATTRIBUTES       */

tiledb/sm/serialization/array_directory.cc

@@ -239,29 +239,33 @@ shared_ptr<ArrayDirectory> array_directory_from_capnp(
 
   // Get unfiltered fragment uris
   if (array_directory_reader.hasUnfilteredFragmentUris()) {
-    for (auto uri : array_directory_reader.getUnfilteredFragmentUris()) {
-      array_directory->unfiltered_fragment_uris().reserve(
-          array_directory_reader.getUnfilteredFragmentUris().size());
+    auto unfiltered_fragment_uris =
+        array_directory_reader.getUnfilteredFragmentUris();
+    array_directory->unfiltered_fragment_uris().reserve(
+        unfiltered_fragment_uris.size());
+    for (auto uri : unfiltered_fragment_uris) {
       array_directory->unfiltered_fragment_uris().emplace_back(
           deserialize_array_uri_to_absolute(uri.cStr(), array_uri));
     }
   }
 
   // Get consolidated commit uris
   if (array_directory_reader.hasConsolidatedCommitUris()) {
-    for (auto uri : array_directory_reader.getConsolidatedCommitUris()) {
-      array_directory->consolidated_commit_uris_set().reserve(
-          array_directory_reader.getConsolidatedCommitUris().size());
+    auto consolidated_commit_uris =
+        array_directory_reader.getConsolidatedCommitUris();
+    array_directory->consolidated_commit_uris_set().reserve(
+        consolidated_commit_uris.size());
+    for (auto uri : consolidated_commit_uris) {
       array_directory->consolidated_commit_uris_set().emplace(
           deserialize_array_uri_to_absolute(uri.cStr(), array_uri).to_string());
     }
   }
 
   // Get array schema uris
   if (array_directory_reader.hasArraySchemaUris()) {
-    for (auto uri : array_directory_reader.getArraySchemaUris()) {
-      array_directory->array_schema_uris().reserve(
-          array_directory_reader.getArraySchemaUris().size());
+    auto array_schema_uris = array_directory_reader.getArraySchemaUris();
+    array_directory->array_schema_uris().reserve(array_schema_uris.size());
+    for (auto uri : array_schema_uris) {
       array_directory->array_schema_uris().emplace_back(
           deserialize_array_uri_to_absolute(uri.cStr(), array_uri));
     }
@@ -277,60 +281,68 @@ shared_ptr<ArrayDirectory> array_directory_from_capnp(
 
   // Get array meta uris to vacuum
   if (array_directory_reader.hasArrayMetaUrisToVacuum()) {
-    for (auto uri : array_directory_reader.getArrayMetaUrisToVacuum()) {
-      array_directory->array_meta_uris_to_vacuum().reserve(
-          array_directory_reader.getArrayMetaUrisToVacuum().size());
+    auto array_meta_uris_to_vacuum =
+        array_directory_reader.getArrayMetaUrisToVacuum();
+    array_directory->array_meta_uris_to_vacuum().reserve(
+        array_meta_uris_to_vacuum.size());
+    for (auto uri : array_meta_uris_to_vacuum) {
       array_directory->array_meta_uris_to_vacuum().emplace_back(
           deserialize_array_uri_to_absolute(uri.cStr(), array_uri));
     }
   }
 
   // Get array meta vac uris to vacuum
   if (array_directory_reader.hasArrayMetaVacUrisToVacuum()) {
-    for (auto uri : array_directory_reader.getArrayMetaVacUrisToVacuum()) {
-      array_directory->array_meta_vac_uris_to_vacuum().reserve(
-          array_directory_reader.getArrayMetaVacUrisToVacuum().size());
+    auto array_meta_vac_uris_to_vacuum =
+        array_directory_reader.getArrayMetaVacUrisToVacuum();
+    array_directory->array_meta_vac_uris_to_vacuum().reserve(
+        array_meta_vac_uris_to_vacuum.size());
+    for (auto uri : array_meta_vac_uris_to_vacuum) {
       array_directory->array_meta_vac_uris_to_vacuum().emplace_back(
           deserialize_array_uri_to_absolute(uri.cStr(), array_uri));
     }
   }
 
   // Get commit uris to consolidate
   if (array_directory_reader.hasCommitUrisToConsolidate()) {
-    for (auto uri : array_directory_reader.getCommitUrisToConsolidate()) {
-      array_directory->commit_uris_to_consolidate().reserve(
-          array_directory_reader.getCommitUrisToConsolidate().size());
+    auto commit_uris_to_consolidate =
+        array_directory_reader.getCommitUrisToConsolidate();
+    array_directory->commit_uris_to_consolidate().reserve(
+        commit_uris_to_consolidate.size());
+    for (auto uri : commit_uris_to_consolidate) {
       array_directory->commit_uris_to_consolidate().emplace_back(
           deserialize_array_uri_to_absolute(uri.cStr(), array_uri));
     }
   }
 
   // Get commit uris to vacuum
   if (array_directory_reader.hasCommitUrisToVacuum()) {
-    for (auto uri : array_directory_reader.getCommitUrisToVacuum()) {
-      array_directory->commit_uris_to_vacuum().reserve(
-          array_directory_reader.getCommitUrisToVacuum().size());
+    auto commit_uris_to_vacuum = array_directory_reader.getCommitUrisToVacuum();
+    array_directory->commit_uris_to_vacuum().reserve(
+        commit_uris_to_vacuum.size());
+    for (auto uri : commit_uris_to_vacuum) {
       array_directory->commit_uris_to_vacuum().emplace_back(
           deserialize_array_uri_to_absolute(uri.cStr(), array_uri));
     }
   }
 
   // Get consolidated commit uris to vacuum
   if (array_directory_reader.hasConsolidatedCommitUrisToVacuum()) {
-    for (auto uri :
-         array_directory_reader.getConsolidatedCommitUrisToVacuum()) {
-      array_directory->consolidated_commits_uris_to_vacuum().reserve(
-          array_directory_reader.getConsolidatedCommitUrisToVacuum().size());
+    auto consolidated_commits_uris_to_vacuum =
+        array_directory_reader.getConsolidatedCommitUrisToVacuum();
+    array_directory->consolidated_commits_uris_to_vacuum().reserve(
+        consolidated_commits_uris_to_vacuum.size());
+    for (auto uri : consolidated_commits_uris_to_vacuum) {
       array_directory->consolidated_commits_uris_to_vacuum().emplace_back(
           deserialize_array_uri_to_absolute(uri.cStr(), array_uri));
     }
   }
 
   // Get array meta uris
   if (array_directory_reader.hasArrayMetaUris()) {
-    for (auto timestamp_reader : array_directory_reader.getArrayMetaUris()) {
-      array_directory->array_meta_uris().reserve(
-          array_directory_reader.getArrayMetaUris().size());
+    auto array_meta_uris = array_directory_reader.getArrayMetaUris();
+    array_directory->array_meta_uris().reserve(array_meta_uris.size());
+    for (auto timestamp_reader : array_meta_uris) {
       array_directory->array_meta_uris().emplace_back(
           URI(deserialize_array_uri_to_absolute(
               timestamp_reader.getUri().cStr(), array_uri)),
@@ -342,20 +354,21 @@ shared_ptr<ArrayDirectory> array_directory_from_capnp(
 
   // Get fragment meta uris
   if (array_directory_reader.hasFragmentMetaUris()) {
-    for (auto uri : array_directory_reader.getFragmentMetaUris()) {
-      array_directory->fragment_meta_uris().reserve(
-          array_directory_reader.getFragmentMetaUris().size());
+    auto fragment_meta_uris = array_directory_reader.getFragmentMetaUris();
+    array_directory->fragment_meta_uris().reserve(fragment_meta_uris.size());
+    for (auto uri : fragment_meta_uris) {
       array_directory->fragment_meta_uris().emplace_back(
           deserialize_array_uri_to_absolute(uri.cStr(), array_uri));
     }
   }
 
   // Get delete tiles location
   if (array_directory_reader.hasDeleteAndUpdateTileLocation()) {
-    for (auto del_tile_reader :
-         array_directory_reader.getDeleteAndUpdateTileLocation()) {
-      array_directory->delete_and_update_tiles_location().reserve(
-          array_directory_reader.getDeleteAndUpdateTileLocation().size());
+    auto delete_and_update_tiles_location =
+        array_directory_reader.getDeleteAndUpdateTileLocation();
+    array_directory->delete_and_update_tiles_location().reserve(
+        delete_and_update_tiles_location.size());
+    for (auto del_tile_reader : delete_and_update_tiles_location) {
       array_directory->delete_and_update_tiles_location().emplace_back(
           URI(deserialize_array_uri_to_absolute(
               del_tile_reader.getUri().cStr(), array_uri)),