Serialization and C API handler changes.

Author: shaunrd0

test/src/unit-request-handlers.cc

@@ -32,6 +32,7 @@
 
 #ifdef TILEDB_SERIALIZATION
 
+#include "test/support/src/helpers.h"
 #include "test/support/src/mem_helpers.h"
 #include "test/support/tdb_catch.h"
 #include "tiledb/api/c_api/buffer/buffer_api_internal.h"
@@ -41,6 +42,7 @@
 #include "tiledb/sm/c_api/tiledb_serialization.h"
 #include "tiledb/sm/c_api/tiledb_struct_def.h"
 #include "tiledb/sm/cpp_api/tiledb"
+#include "tiledb/sm/cpp_api/tiledb_experimental"
 #include "tiledb/sm/crypto/encryption_key.h"
 #include "tiledb/sm/enums/array_type.h"
 #include "tiledb/sm/enums/encryption_type.h"
@@ -67,6 +69,7 @@ struct RequestHandlerFx {
   Config cfg_;
   Context ctx_;
   EncryptionKey enc_key_;
+  shared_ptr<ArraySchema> schema_;
 };
 
 struct HandleLoadArraySchemaRequestFx : RequestHandlerFx {
@@ -75,11 +78,17 @@ struct HandleLoadArraySchemaRequestFx : RequestHandlerFx {
   }
 
   virtual shared_ptr<ArraySchema> create_schema() override;
-  shared_ptr<ArraySchema> call_handler(
+
+  std::tuple<
+      shared_ptr<ArraySchema>,
+      std::unordered_map<std::string, shared_ptr<ArraySchema>>>
+  call_handler(
       serialization::LoadArraySchemaRequest req, SerializationType stype);
 
   shared_ptr<const Enumeration> create_string_enumeration(
       std::string name, std::vector<std::string>& values);
+
+  shared_ptr<ArraySchema> schema_add_attribute(const std::string& attr_name);
 };
 
 struct HandleQueryPlanRequestFx : RequestHandlerFx {
@@ -116,15 +125,23 @@ struct HandleConsolidationPlanRequestFx : RequestHandlerFx {
 
 TEST_CASE_METHOD(
     HandleLoadArraySchemaRequestFx,
-    "tiledb_handle_load_array_schema_request - default request",
+    "tiledb_handle_load_array_schema_request - no enumerations",
     "[request_handler][load_array_schema][default]") {
   auto stype = GENERATE(SerializationType::JSON, SerializationType::CAPNP);
 
   create_array();
-  auto schema =
-      call_handler(serialization::LoadArraySchemaRequest(false), stype);
+  auto schema_response =
+      call_handler(serialization::LoadArraySchemaRequest(cfg_), stype);
+  auto schema = std::get<0>(schema_response);
   REQUIRE(schema->has_enumeration("enmr"));
   REQUIRE(schema->get_loaded_enumeration_names().size() == 0);
+  tiledb::test::schema_equiv(*schema, *schema_);
+
+  // We did not evolve the schema so there should only be one.
+  auto all_schemas = std::get<1>(schema_response);
+  REQUIRE(all_schemas.size() == 1);
+  tiledb::test::schema_equiv(
+      *all_schemas.find(schema->name())->second, *schema_);
 }
 
 TEST_CASE_METHOD(
@@ -134,12 +151,57 @@ TEST_CASE_METHOD(
   auto stype = GENERATE(SerializationType::JSON, SerializationType::CAPNP);
 
   create_array();
-  auto schema =
-      call_handler(serialization::LoadArraySchemaRequest(true), stype);
+  REQUIRE(cfg_.set("rest.load_enumerations_on_array_open", "true").ok());
+  auto schema_response =
+      call_handler(serialization::LoadArraySchemaRequest(cfg_), stype);
+  auto schema = std::get<0>(schema_response);
   REQUIRE(schema->has_enumeration("enmr"));
   REQUIRE(schema->get_loaded_enumeration_names().size() == 1);
   REQUIRE(schema->get_loaded_enumeration_names()[0] == "enmr");
   REQUIRE(schema->get_enumeration("enmr") != nullptr);
+  tiledb::test::schema_equiv(*schema, *schema_);
+
+  // We did not evolve the schema so there should only be one.
+  auto all_schemas = std::get<1>(schema_response);
+  REQUIRE(all_schemas.size() == 1);
+  tiledb::test::schema_equiv(
+      *all_schemas.find(schema->name())->second, *schema_);
+}
+
+TEST_CASE_METHOD(
+    HandleLoadArraySchemaRequestFx,
+    "tiledb_handle_load_array_schema_request - multiple schemas",
+    "[request_handler][load_array_schema][schema-evolution]") {
+  auto stype = GENERATE(SerializationType::JSON, SerializationType::CAPNP);
+  std::string load_enums = GENERATE("true", "false");
+
+  create_array();
+
+  std::vector<shared_ptr<ArraySchema>> all_schemas{schema_};
+  all_schemas.push_back(schema_add_attribute("b"));
+  all_schemas.push_back(schema_add_attribute("c"));
+  all_schemas.push_back(schema_add_attribute("d"));
+
+  REQUIRE(cfg_.set("rest.load_enumerations_on_array_open", load_enums).ok());
+  auto schema_response =
+      call_handler(serialization::LoadArraySchemaRequest(cfg_), stype);
+  auto schema = std::get<0>(schema_response);
+  if (load_enums == "true") {
+    REQUIRE(schema->has_enumeration("enmr"));
+    REQUIRE(schema->get_loaded_enumeration_names().size() == 1);
+    REQUIRE(schema->get_loaded_enumeration_names()[0] == "enmr");
+    REQUIRE(schema->get_enumeration("enmr") != nullptr);
+  }
+  // The latest schema should be equal to the last applied evolution.
+  tiledb::test::schema_equiv(*schema, *all_schemas.back());
+
+  // Validate schemas returned from the request in the order they were created.
+  auto r_all_schemas = std::get<1>(schema_response);
+  std::map<std::string, shared_ptr<ArraySchema>> resp(
+      r_all_schemas.begin(), r_all_schemas.end());
+  for (int i = 0; const auto& s : resp) {
+    tiledb::test::schema_equiv(*s.second, *all_schemas[i++]);
+  }
 }
 
 TEST_CASE_METHOD(
@@ -346,7 +408,9 @@ TEST_CASE_METHOD(
 RequestHandlerFx::RequestHandlerFx(const std::string uri)
     : memory_tracker_(tiledb::test::create_test_memory_tracker())
     , uri_(uri)
-    , ctx_(cfg_) {
+    , ctx_(cfg_)
+    , schema_(make_shared<ArraySchema>(
+          ArrayType::DENSE, ctx_.resources().ephemeral_memory_tracker())) {
   delete_array();
   throw_if_not_ok(enc_key_.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0));
 }
@@ -405,9 +469,28 @@ HandleLoadArraySchemaRequestFx::create_string_enumeration(
       tiledb::test::create_test_memory_tracker());
 }
 
+shared_ptr<ArraySchema> HandleLoadArraySchemaRequestFx::schema_add_attribute(
+    const std::string& attr_name) {
+  tiledb::Context ctx;
+  tiledb::ArraySchemaEvolution ase(ctx);
+  auto attr = tiledb::Attribute::create<int32_t>(ctx, attr_name);
+  ase.add_attribute(attr);
+  // Evolve and update the original schema member variable.
+  schema_ = ase.ptr()->array_schema_evolution_->evolve_schema(schema_);
+  // Apply the schema evolution.
+  Array::evolve_array_schema(
+      this->ctx_.resources(),
+      this->uri_,
+      ase.ptr()->array_schema_evolution_,
+      this->enc_key_);
+
+  // Return the new evolved schema for validation.
+  return schema_;
+}
+
 shared_ptr<ArraySchema> HandleLoadArraySchemaRequestFx::create_schema() {
   // Create a schema to serialize
-  auto schema =
+  schema_ =
       make_shared<ArraySchema>(HERE(), ArrayType::SPARSE, memory_tracker_);
   auto dim =
       make_shared<Dimension>(HERE(), "dim1", Datatype::INT32, memory_tracker_);
@@ -416,20 +499,23 @@ shared_ptr<ArraySchema> HandleLoadArraySchemaRequestFx::create_schema() {
 
   auto dom = make_shared<Domain>(HERE(), memory_tracker_);
   throw_if_not_ok(dom->add_dimension(dim));
-  throw_if_not_ok(schema->set_domain(dom));
+  throw_if_not_ok(schema_->set_domain(dom));
 
   std::vector<std::string> values = {"pig", "cow", "chicken", "dog", "cat"};
   auto enmr = create_string_enumeration("enmr", values);
-  schema->add_enumeration(enmr);
+  schema_->add_enumeration(enmr);
 
   auto attr = make_shared<Attribute>(HERE(), "attr", Datatype::INT32);
   attr->set_enumeration_name("enmr");
-  throw_if_not_ok(schema->add_attribute(attr));
+  throw_if_not_ok(schema_->add_attribute(attr));
 
-  return schema;
+  return schema_;
 }
 
-shared_ptr<ArraySchema> HandleLoadArraySchemaRequestFx::call_handler(
+std::tuple<
+    shared_ptr<ArraySchema>,
+    std::unordered_map<std::string, shared_ptr<ArraySchema>>>
+HandleLoadArraySchemaRequestFx::call_handler(
     serialization::LoadArraySchemaRequest req, SerializationType stype) {
   // If this looks weird, its because we're using the public C++ API to create
   // these objets instead of the internal APIs elsewhere in this test suite.
@@ -451,7 +537,7 @@ shared_ptr<ArraySchema> HandleLoadArraySchemaRequestFx::call_handler(
   REQUIRE(rval == TILEDB_OK);
 
   return serialization::deserialize_load_array_schema_response(
-      stype, resp_buf->buffer(), memory_tracker_);
+      uri_, stype, resp_buf->buffer(), memory_tracker_);
 }
 
 shared_ptr<ArraySchema> HandleQueryPlanRequestFx::create_schema() {

test/support/src/helpers.cc

@@ -1621,6 +1621,26 @@ void read_sparse_v11(
   tiledb_query_free(&query);
 }
 
+void schema_equiv(
+    const sm::ArraySchema& schema1, const sm::ArraySchema& schema2) {
+  CHECK(schema1.array_type() == schema2.array_type());
+  CHECK(schema1.attributes().size() == schema2.attributes().size());
+  for (unsigned int i = 0; i < schema2.attribute_num(); i++) {
+    auto a = schema1.attribute(i);
+    auto b = schema2.attribute(i);
+    CHECK(a->cell_val_num() == b->cell_val_num());
+    CHECK(a->name() == b->name());
+    CHECK(a->type() == b->type());
+    CHECK(a->nullable() == b->nullable());
+    CHECK(a->get_enumeration_name() == b->get_enumeration_name());
+  }
+  CHECK(schema1.capacity() == schema2.capacity());
+  CHECK(schema1.cell_order() == schema2.cell_order());
+  CHECK(schema1.tile_order() == schema2.tile_order());
+  CHECK(schema1.allows_dups() == schema2.allows_dups());
+  CHECK(schema1.array_uri().to_string() == schema2.array_uri().to_string());
+}
+
 template void check_subarray<int8_t>(
     tiledb::sm::Subarray& subarray, const SubarrayRanges<int8_t>& ranges);
 template void check_subarray<uint8_t>(

test/support/src/helpers.h

@@ -957,6 +957,15 @@ void write_sparse_v11(
  */
 void read_sparse_v11(
     tiledb_ctx_t* ctx, const std::string& array_name, uint64_t timestamp);
+
+/**
+ * Helper function to test two array schemas are equivalent.
+ *
+ * @param schema1 Expected array schema.
+ * @param schema2 Actual array schema.
+ */
+void schema_equiv(
+    const sm::ArraySchema& schema1, const sm::ArraySchema& schema2);
 }  // namespace tiledb::test
 
 #endif

tiledb/sm/c_api/tiledb.cc

@@ -3894,12 +3894,13 @@ capi_return_t tiledb_handle_load_array_schema_request(
     tiledb_serialization_type_t serialization_type,
     const tiledb_buffer_t* request,
     tiledb_buffer_t* response) {
+  ensure_context_is_valid(ctx);
   if (sanity_check(ctx, array) == TILEDB_ERR) {
-    throw std::invalid_argument("Array paramter must be valid.");
+    throw CAPIStatusException("Array paramter must be valid.");
   }
-
-  api::ensure_buffer_is_valid(request);
-  api::ensure_buffer_is_valid(response);
+  ensure_array_is_valid(array);
+  ensure_buffer_is_valid(request);
+  ensure_buffer_is_valid(response);
 
   auto load_schema_req =
       tiledb::sm::serialization::deserialize_load_array_schema_request(
@@ -3911,7 +3912,7 @@ capi_return_t tiledb_handle_load_array_schema_request(
   }
 
   tiledb::sm::serialization::serialize_load_array_schema_response(
-      array->array_->array_schema_latest(),
+      *array->array_,
       static_cast<tiledb::sm::SerializationType>(serialization_type),
       response->buffer());
 

tiledb/sm/rest/rest_client.h

@@ -317,7 +317,10 @@ class RestClient {
   }
 
   /// Operation disabled in base class.
-  inline virtual shared_ptr<ArraySchema> post_array_schema_from_rest(
+  inline virtual std::tuple<
+      shared_ptr<ArraySchema>,
+      std::unordered_map<std::string, shared_ptr<ArraySchema>>>
+  post_array_schema_from_rest(
       const Config&, const URI&, uint64_t, uint64_t, bool) {
     throw RestClientDisabledException();
   }

tiledb/sm/rest/rest_client_remote.cc

@@ -226,13 +226,16 @@ RestClientRemote::get_array_schema_from_rest(const URI& uri) {
           serialization_type_, returned_data, memory_tracker_)};
 }
 
-shared_ptr<ArraySchema> RestClientRemote::post_array_schema_from_rest(
+std::tuple<
+    shared_ptr<ArraySchema>,
+    std::unordered_map<std::string, shared_ptr<ArraySchema>>>
+RestClientRemote::post_array_schema_from_rest(
     const Config& config,
     const URI& uri,
     uint64_t timestamp_start,
     uint64_t timestamp_end,
     bool include_enumerations) {
-  serialization::LoadArraySchemaRequest req(include_enumerations);
+  serialization::LoadArraySchemaRequest req(config);
 
   Buffer buf;
   serialization::serialize_load_array_schema_request(
@@ -271,7 +274,7 @@ shared_ptr<ArraySchema> RestClientRemote::post_array_schema_from_rest(
   // Ensure data has a null delimiter for cap'n proto if using JSON
   throw_if_not_ok(ensure_json_null_delimited_string(&returned_data));
   return serialization::deserialize_load_array_schema_response(
-      serialization_type_, returned_data, memory_tracker_);
+      uri, serialization_type_, returned_data, memory_tracker_);
 }
 
 Status RestClientRemote::post_array_schema_to_rest(

tiledb/sm/rest/rest_client_remote.h

@@ -151,10 +151,17 @@ class RestClientRemote : public RestClient {
    * get_array_schema_from_rest after TileDB-Cloud-REST merges support for the
    * POST endpoint.
    *
+   * @param config The TileDB config.
    * @param uri The Array URI to load the schema from.
-   * @return shared_ptr<ArraySchema> The loaded array schema.
-   */
-  shared_ptr<ArraySchema> post_array_schema_from_rest(
+   * @param timestamp_start The starting timestamp used to open the array.
+   * @param timestamp_end The ending timestamp used to open the array.
+   * @return Tuple containing the latest array schema, and all array schemas for
+   *    the array opened with provided timestamps.
+   */
+  std::tuple<
+      shared_ptr<ArraySchema>,
+      std::unordered_map<std::string, shared_ptr<ArraySchema>>>
+  post_array_schema_from_rest(
       const Config& config,
       const URI& uri,
       uint64_t timestamp_start,