support dimension domains that don't start from 0

Author: DimitrisStaratzis

test/src/unit-cppapi-max-fragment-size.cc

@@ -613,12 +613,12 @@ TEST_CASE(
   cleanup();
 
   // Remove the array at the end of this test.
-  // ScopedExecutor deferred(cleanup);
+  ScopedExecutor deferred(cleanup);
 
   // Create an array with exactly 9 tiles and tile extend 1
   Domain domain(ctx);
   ArraySchema schema(ctx, TILEDB_DENSE);
-  auto d1 = tiledb::Dimension::create<int32_t>(ctx, "d1", {{0, 8}}, 3);
+  auto d1 = tiledb::Dimension::create<int32_t>(ctx, "d1", {{1, 9}}, 3);
   domain.add_dimension(d1);
 
   auto a1 = tiledb::Attribute::create<int32_t>(ctx, "a");
@@ -631,7 +631,7 @@ TEST_CASE(
 
   // Populate array with data from 1 to 9
   int value = 0;
-  for (int i = 0; i < 9; i += 3) {
+  for (int i = 1; i < 10; i += 3) {
     Array array(ctx, array_name, TILEDB_WRITE);
     Query query(ctx, array);
     query.set_layout(TILEDB_ROW_MAJOR);
@@ -649,7 +649,7 @@ TEST_CASE(
 
   Array array(ctx, array_name, TILEDB_READ);
   tiledb::Subarray sub(ctx, array);
-  sub.set_subarray({0, 8});
+  sub.set_subarray({1, 9});
   std::vector<int32_t> a(9);
   Query query(ctx, array, TILEDB_READ);
   query.set_subarray(sub).set_layout(TILEDB_ROW_MAJOR).set_data_buffer("a", a);
@@ -674,7 +674,7 @@ TEST_CASE(
   // Read data to validate correctness
   Array array2(ctx, array_name, TILEDB_READ);
   tiledb::Subarray sub2(ctx, array2);
-  sub2.set_subarray({0, 8});
+  sub2.set_subarray({1, 9});
   std::vector<int32_t> a2(9);
   Query query2(ctx, array2, TILEDB_READ);
   query2.set_subarray(sub2)

tiledb/sm/query/writers/global_order_writer.cc

@@ -94,7 +94,8 @@ GlobalOrderWriter::GlobalOrderWriter(
     , processed_conditions_(processed_conditions)
     , fragment_size_(fragment_size)
     , current_fragment_size_(0)
-    , rows_written_(0) {
+    , rows_written_(0)
+    , start_(0) {
   // Check the layout is global order.
   if (layout_ != Layout::GLOBAL_ORDER) {
     throw GlobalOrderWriterException(
@@ -1463,16 +1464,6 @@ NDRange GlobalOrderWriter::ndranges_after_split(uint64_t num) {
 
   // Calculate how many tiles each row can hold
   uint64_t tiles_per_row = num_tiles_per_row(domain);
-  auto dim_num = domain.dim_num();
-  auto dim{domain.dimension_ptr(0)};
-
-  auto l = [&](auto T) {
-    return static_cast<uint64_t>(dim->tile_extent().rvalue_as<decltype(T)>());
-  };
-
-  uint64_t tile_extent = apply_with_type(l, dim->type());
-  NDRange nd;
-  nd.reserve(dim_num);
 
   if (num % tiles_per_row != 0) {
     throw GlobalOrderWriterException(
@@ -1484,13 +1475,33 @@ NDRange GlobalOrderWriter::ndranges_after_split(uint64_t num) {
   // Calculate how many rows we will write in the current fragment
   uint64_t rows_of_tiles_to_write = num / tiles_per_row;
 
-  // Create the range for the index dim (first).
-  int start = rows_written_ * tile_extent;
-  int end = start + (rows_of_tiles_to_write * tile_extent) - 1;
-  Range range(&start, &end, sizeof(int));
+  // Create NDRange object and reserve for dims
+  auto dim_num = domain.dim_num();
+  NDRange nd;
+  nd.reserve(dim_num);
+
+  // Calculate the range for the index dim (first).
+  auto dim{domain.dimension_ptr(0)};
+  auto dim_dom = dim->domain();
+  auto l = [&](auto T) {
+    return static_cast<uint64_t>(dim->tile_extent().rvalue_as<decltype(T)>());
+  };
+  uint64_t tile_extent = apply_with_type(l, dim->type());
+
+  // Calculate start and end
+  if (rows_written_ == 0) {
+    // It means that the start has not been set yet. Set it to the minimum value
+    // of the expanded domain for that dim
+    auto dim_dom_data = (const uint64_t*)dim_dom.data();
+    start_ = dim_dom_data[0];
+  }
+  uint64_t end = start_ + (rows_of_tiles_to_write * tile_extent) - 1;
+
+  // Add range
+  Range range(&start_, &end, sizeof(int));
   nd.emplace_back(range);
 
-  // Use the domain as ranges for the rest of the dims
+  // For the rest of the dims, use their domains as ranges. No split there.
   for (unsigned d = 1; d < dim_num; ++d) {
     // begin from second dim
     auto dim{array_schema_.dimension_ptr(d)};
@@ -1500,6 +1511,7 @@ NDRange GlobalOrderWriter::ndranges_after_split(uint64_t num) {
 
   // add rows written to the cache
   rows_written_ += rows_of_tiles_to_write;
+  start_ = end + 1;
 
   return nd;
 }

tiledb/sm/query/writers/global_order_writer.h

@@ -221,6 +221,12 @@ class GlobalOrderWriter : public WriterBase {
    */
   uint64_t rows_written_;
 
+  /**
+   * This is the start for the dim range in case we need to split in multiple
+   * fragments in Dense arrays
+   */
+  uint64_t start_;
+
   /* ********************************* */
   /*           PRIVATE METHODS         */
   /* ********************************* */