fix support for support multiple cells per tile

Author: DimitrisStaratzis

test/src/unit-cppapi-max-fragment-size.cc

@@ -597,3 +597,94 @@ TEST_CASE(
     CHECK(a2[i] == i + 1);
   }
 }
+
+TEST_CASE(
+    "Setting max_fragment_size in Dense consolidation one dim",
+    "[global-order-writer]") {
+  std::string array_name = "cpp_max_fragment_size_bug";
+  Context ctx;
+
+  auto cleanup = [&]() {
+    auto obj = Object::object(ctx, array_name);
+    if (obj.type() == Object::Type::Array) {
+      Object::remove(ctx, array_name);
+    }
+  };
+
+  cleanup();
+
+  // Remove the array at the end of this test.
+  // ScopedExecutor deferred(cleanup);
+
+  // Create an array with exactly 9 tiles and tile extend 1
+  Domain domain(ctx);
+  ArraySchema schema(ctx, TILEDB_DENSE);
+  auto d1 = tiledb::Dimension::create<int32_t>(ctx, "d1", {{0, 8}}, 3);
+  domain.add_dimension(d1);
+
+  auto a1 = tiledb::Attribute::create<int32_t>(ctx, "a");
+  schema.add_attribute(a1);
+
+  schema.set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}});
+  schema.set_domain(domain);
+
+  Array::create(array_name, schema);
+
+  // Populate array with data from 1 to 9
+  int value = 0;
+  for (int i = 0; i < 9; i+=3) {
+    Array array(ctx, array_name, TILEDB_WRITE);
+    Query query(ctx, array);
+    query.set_layout(TILEDB_ROW_MAJOR);
+    tiledb::Subarray sub(ctx, array);
+    sub.set_subarray({i, i + 2});
+    query.set_subarray(sub);
+    std::vector<int32_t> data = {++value, ++value, ++value};
+    query.set_data_buffer("a", data);
+    query.submit();
+    array.close();
+  }
+
+  // Read data to validate write and num of fragments.
+  // CHECK(tiledb::test::num_fragments(array_name) == 3);
+
+  Array array(ctx, array_name, TILEDB_READ);
+  tiledb::Subarray sub(ctx, array);
+  sub.set_subarray({0, 8});
+  std::vector<int32_t> a(9);
+  Query query(ctx, array, TILEDB_READ);
+  query.set_subarray(sub).set_layout(TILEDB_ROW_MAJOR).set_data_buffer("a", a);
+  query.submit();
+  array.close();
+
+  for (int i = 0; i < 9; i++) {
+    CHECK(a[i] == i + 1);
+  }
+
+  // Consolidate with a size limitation for the fragment. This will result in
+  // the creation of two new fragments.
+  tiledb::Config cfg;
+  cfg.set("sm.consolidation.max_fragment_size", "80");
+  ctx = Context(cfg);
+  Array::consolidate(ctx, array_name);
+  Array::vacuum(ctx, array_name);
+
+  // Check that we now have 2 fragments instead of 3
+  CHECK(tiledb::test::num_fragments(array_name) == 2);
+
+  // Read data to validate correctness
+  Array array2(ctx, array_name, TILEDB_READ);
+  tiledb::Subarray sub2(ctx, array2);
+  sub2.set_subarray({0, 8});
+  std::vector<int32_t> a2(9);
+  Query query2(ctx, array2, TILEDB_READ);
+  query2.set_subarray(sub2)
+      .set_layout(TILEDB_ROW_MAJOR)
+      .set_data_buffer("a", a2);
+  query2.submit();
+  array2.close();
+
+  for (int i = 0; i < 9; i++) {
+    CHECK(a2[i] == i + 1);
+  }
+}

tiledb/sm/query/writers/global_order_writer.cc

@@ -1435,8 +1435,11 @@ uint64_t GlobalOrderWriter::num_tiles_to_write(
 uint64_t GlobalOrderWriter::num_tiles_per_row() {
   auto dim_num = array_schema_.dim_num();
   uint64_t ret = 1;
+
   for (unsigned d = 1; d < dim_num; ++d) {
-    // skip first dim. todo Explain
+    // Skip first dim. We want to calculate how many tiles can fit in one row.
+    // To do that we skip the first dim and multiply the range / extend of the
+    // other dimensions
     auto dim{array_schema_.dimension_ptr(d)};
     auto dim_dom = dim->domain();
     ret *= dim->domain_range(dim_dom) / dim->tile_extent().rvalue_as<int32_t>();
@@ -1449,20 +1452,24 @@ uint64_t GlobalOrderWriter::num_tiles_per_row() {
 NDRange GlobalOrderWriter::ndranges_after_split(uint64_t num) {
   uint64_t tiles_per_row = num_tiles_per_row();
   auto dim_num = array_schema_.dim_num();
+  uint64_t cells_in_tile = array_schema_.domain().cell_num_per_tile();
   NDRange nd;
   nd.reserve(dim_num);
 
   if (num % tiles_per_row != 0) {
     throw GlobalOrderWriterException(
-        "This fragment target size is not possible please try something else ");  // todo fix
+        "The target fragment size cannot be achieved. Please try using a "
+        "different size, or there might be a misconfiguration in the array "
+        "schema.");
   }
 
   // Calculate how many rows we will write in the current fragment
-  uint64_t rows_to_write = num / tiles_per_row;
+  uint64_t rows_of_tiles_to_write = num / tiles_per_row;
 
   // Create the range for the index dim (first).
-  int start = rows_written_;
-  int end = start + rows_to_write - 1;
+  int start =
+      rows_written_ * cells_in_tile;  // todo start from the dim_dom start
+  int end = start + (rows_of_tiles_to_write * cells_in_tile) - 1;
   Range range(&start, &end, sizeof(int));
   nd.emplace_back(range);
 
@@ -1475,7 +1482,7 @@ NDRange GlobalOrderWriter::ndranges_after_split(uint64_t num) {
   }
 
   // add rows written to the cache
-  rows_written_ += rows_to_write;
+  rows_written_ += rows_of_tiles_to_write;
 
   return nd;
 }

tiledb/sm/query/writers/global_order_writer.h

@@ -392,12 +392,19 @@ class GlobalOrderWriter : public WriterBase {
       tdb::pmr::unordered_map<std::string, WriterTileTupleVector>& tiles);
 
   /**
-   * Return the number of tiles a single row can hold
+   * Create new ndranges by splitting the first dimension based on the number of
+   * tiles we need to write
+   * @param num The number of tiles we need to write.
    *
-   * @return Number of tiles.
    */
   NDRange ndranges_after_split(uint64_t num);
 
+  /**
+   * Return the number of tiles a single row can hold. More specifically, the
+   * number of tiles all dimensions except the first can hold.
+   *
+   * @return Number of tiles.
+   */
   uint64_t num_tiles_per_row();
 
   /**