chore: Add global order min/max per tile to the fragment metadata

format_spec/FORMAT_SPEC.md

@@ -4,7 +4,7 @@ title: Format Specification
 
 **Notes:**
 
-* The current TileDB array format version number is **22** (`uint32_t`).
+* The current TileDB array format version number is **23** (`uint32_t`).
   * Other structures might be versioned separately.
 * Data written by TileDB and referenced in this document is **little-endian**
   with the following exceptions:

format_spec/array_format_history.md

@@ -4,6 +4,12 @@ title: Array format version history
 
 # Array Format Version History
 
+## Version 23
+
+Introduced in TileDB 2.30
+
+* The _Tile global order min/max_ fields were added to [tile metadata](./fragment.md#tile-mins-maxes). These fields contain the minimum and maximum global order coordinate for each tile in the fragment. This metadata can be used to optimize query execution.
+
 ## Version 22
 
 Introduced in TileDB 2.25

format_spec/fragment.md

@@ -82,6 +82,12 @@ The fragment metadata file has the following on-disk format:
 | Tile maxes for attribute/dimension 1 | [Tile Mins/Maxes](#tile-mins-maxes) | _New in version 11_ The serialized maxes for attribute/dimension 1 |
 | … | … | … |
 | Variable maxes for attribute/dimension N | [Tile Mins/Maxes](#tile-mins-maxes) | _New in version 11_ The serialized maxes for attribute/dimension N |
+| Tile global order min coordinates for dimension 1 | [Tile Mins/Maxes](#tile-mins-maxes) | _New in version 23_ For sparse arrays, the serialized value of dimension 1 of the global order minimum coordinate in each tile. |
+| … | … | … |
+| Variable global order min coordinates for dimension N | [Tile Mins/Maxes](#tile-mins-maxes) | _New in version 23_ For sparse arrays, the serialized value of dimension N of the global order minimum coordinate in each tile. |
+| Tile global order max coordinates for dimension 1 | [Tile Mins/Maxes](#tile-mins-maxes) | _New in version 23_ For sparse arrays, the serialized value of dimension 1 of the global order maximum coordinate in each tile. |
+| … | … | … |
+| Variable global order max coordinates for dimension N | [Tile Mins/Maxes](#tile-mins-maxes) | _New in version 23_ For sparse arrays, the serialized value of dimension N of the global order maximum coordinate in each tile. |
 | Tile sums for attribute/dimension 1 | [Tile Sums](#tile-sums) | _New in version 11_ The serialized sums for attribute/dimension 1 |
 | … | … | … |
 | Variable sums for attribute/dimension N | [Tile Sums](#tile-sums) | _New in version 11_ The serialized sums for attribute/dimension N |
@@ -276,6 +282,12 @@ The footer is a simple blob \(i.e., _not a generic tile_\) with the following in
 | Tile maxes offset for attribute/dimension 1 | `uint64_t` | The offset to the generic tile storing the tile maxes for attribute/dimension 1. |
 | … | … | … |
 | Tile maxes offset for attribute/dimension N | `uint64_t` | The offset to the generic tile storing the tile maxes for attribute/dimension N |
+| Tile global order min coordinates offset for dimension 1 | `uint64_t` | _New in version 23_ For sparse arrays, the offset to the generic tile storing the tile global order mins for dimension 1
+| … | … | … |
+| Tile global order min coordinates offset for dimension N | `uint64_t` | _New in version 23_ For sparse arrays, the offset to the generic tile storing the tile global order mins for dimension N
+| Tile global order max coordinates offset for dimension 1 | `uint64_t` | _New in version 23_ For sparse arrays, the offset to the generic tile storing the tile global order maxes for dimension 1
+| … | … | … |
+| Tile global order max coordinates offset for dimension N | `uint64_t` | _New in version 23_ For sparse arrays, the offset to the generic tile storing the tile global order maxes for dimension N
 | Tile sums offset for attribute/dimension 1 | `uint64_t` | The offset to the generic tile storing the tile sums for attribute/dimension 1. |
 | … | … | … |
 | Tile sums offset for attribute/dimension N | `uint64_t` | The offset to the generic tile storing the tile sums for attribute/dimension N |

test/CMakeLists.txt

@@ -110,6 +110,7 @@ set(TILEDB_UNIT_TEST_SOURCES
   src/unit-enumerations.cc
   src/unit-enum-helpers.cc
   src/unit-filter-buffer.cc
+  src/unit-fragment-info-global-order-bounds.cc
   src/unit-global-order.cc
   src/unit-ordered-dim-label-reader.cc
   src/unit-tile-metadata.cc

test/src/unit-capi-fragment_info.cc

@@ -30,6 +30,7 @@
  * Tests the C API functions for manipulating fragment information.
  */
 
+#include "test/support/src/error_helpers.h"
 #include "test/support/src/helpers.h"
 #include "test/support/src/serialization_wrappers.h"
 #include "tiledb/sm/c_api/tiledb.h"
@@ -553,6 +554,22 @@ TEST_CASE(
       ctx, fragment_info, 1, 0, "d", &mbr[0]);
   CHECK(rc == TILEDB_ERR);
 
+  // Get global order lower bound - should fail since it's a dense array
+  {
+    void* dimensions[] = {&mbr[0], &mbr[1]};
+    rc = tiledb_fragment_info_get_global_order_lower_bound(
+        ctx, fragment_info, 0, 0, nullptr, &dimensions[0]);
+    CHECK(rc == TILEDB_ERR);
+  }
+
+  // Get global order upper bound - should fail since it's a dense array
+  {
+    void* dimensions[] = {&mbr[0], &mbr[1]};
+    rc = tiledb_fragment_info_get_global_order_upper_bound(
+        ctx, fragment_info, 0, 0, nullptr, &dimensions[0]);
+    CHECK(rc == TILEDB_ERR);
+  }
+
   // Get version
   uint32_t version;
   rc = tiledb_fragment_info_get_version(ctx, fragment_info, 0, &version);
@@ -703,13 +720,14 @@ TEST_CASE("C API: Test MBR fragment info", "[capi][fragment_info][mbr]") {
   // Load fragment info
   rc = tiledb_fragment_info_load(ctx, fragment_info);
   CHECK(rc == TILEDB_OK);
-  tiledb_config_free(&cfg);
 
   tiledb_fragment_info_t* deserialized_fragment_info = nullptr;
   if (serialized_load) {
     rc = tiledb_fragment_info_alloc(
         ctx, array_name.c_str(), &deserialized_fragment_info);
     CHECK(rc == TILEDB_OK);
+    rc = tiledb_fragment_info_set_config(ctx, deserialized_fragment_info, cfg);
+    CHECK(rc == TILEDB_OK);
     tiledb_fragment_info_serialize(
         ctx,
         array_name.c_str(),
@@ -720,6 +738,8 @@ TEST_CASE("C API: Test MBR fragment info", "[capi][fragment_info][mbr]") {
     fragment_info = deserialized_fragment_info;
   }
 
+  tiledb_config_free(&cfg);
+
   // Get fragment num
   uint32_t fragment_num;
   rc = tiledb_fragment_info_get_fragment_num(ctx, fragment_info, &fragment_num);
@@ -731,6 +751,7 @@ TEST_CASE("C API: Test MBR fragment info", "[capi][fragment_info][mbr]") {
   rc = tiledb_fragment_info_get_mbr_num(ctx, fragment_info, 0, &mbr_num);
   CHECK(rc == TILEDB_OK);
   CHECK(mbr_num == 1);
+
   rc = tiledb_fragment_info_get_mbr_num(ctx, fragment_info, 1, &mbr_num);
   CHECK(rc == TILEDB_OK);
   CHECK(mbr_num == 2);
@@ -753,6 +774,108 @@ TEST_CASE("C API: Test MBR fragment info", "[capi][fragment_info][mbr]") {
   CHECK(rc == TILEDB_OK);
   CHECK(mbr == std::vector<uint64_t>{7, 8});
 
+  // Get global order lower bounds
+  {
+    std::vector<uint64_t> lower_bound(2);
+    void* dimensions[] = {&lower_bound[0], &lower_bound[1]};
+
+    // first fragment - one tile
+    rc = tiledb_fragment_info_get_global_order_lower_bound(
+        ctx, fragment_info, 0, 0, nullptr, &dimensions[0]);
+    CHECK(error_if_any(ctx, rc) == std::nullopt);
+    if (rc == TILEDB_OK) {
+      CHECK(lower_bound == std::vector<uint64_t>{1, 1});
+    }
+    rc = tiledb_fragment_info_get_global_order_lower_bound(
+        ctx, fragment_info, 0, 1, nullptr, &dimensions[0]);
+    CHECK(rc == TILEDB_ERR);
+
+    // second fragment - two tiles
+    rc = tiledb_fragment_info_get_global_order_lower_bound(
+        ctx, fragment_info, 1, 0, nullptr, &dimensions[0]);
+    CHECK(error_if_any(ctx, rc) == std::nullopt);
+    if (rc == TILEDB_OK) {
+      CHECK(lower_bound == std::vector<uint64_t>{1, 1});
+    }
+    rc = tiledb_fragment_info_get_global_order_lower_bound(
+        ctx, fragment_info, 1, 1, nullptr, &dimensions[0]);
+    CHECK(error_if_any(ctx, rc) == std::nullopt);
+    if (rc == TILEDB_OK) {
+      CHECK(lower_bound == std::vector<uint64_t>{7, 7});
+    }
+    rc = tiledb_fragment_info_get_global_order_lower_bound(
+        ctx, fragment_info, 1, 2, nullptr, &dimensions[0]);
+    CHECK(rc == TILEDB_ERR);
+
+    // third fragment - two tiles
+    rc = tiledb_fragment_info_get_global_order_lower_bound(
+        ctx, fragment_info, 2, 0, nullptr, &dimensions[0]);
+    CHECK(error_if_any(ctx, rc) == std::nullopt);
+    if (rc == TILEDB_OK) {
+      CHECK(lower_bound == std::vector<uint64_t>{1, 1});
+    }
+    rc = tiledb_fragment_info_get_global_order_lower_bound(
+        ctx, fragment_info, 2, 1, nullptr, &dimensions[0]);
+    CHECK(error_if_any(ctx, rc) == std::nullopt);
+    if (rc == TILEDB_OK) {
+      CHECK(lower_bound == std::vector<uint64_t>{1, 8});
+    }
+    rc = tiledb_fragment_info_get_global_order_lower_bound(
+        ctx, fragment_info, 2, 2, nullptr, &dimensions[0]);
+    CHECK(rc == TILEDB_ERR);
+  }
+
+  // Get global order upper bounds
+  {
+    std::vector<uint64_t> upper_bound(2);
+    void* dimensions[] = {&upper_bound[0], &upper_bound[1]};
+
+    // first fragment - one tile
+    rc = tiledb_fragment_info_get_global_order_upper_bound(
+        ctx, fragment_info, 0, 0, nullptr, &dimensions[0]);
+    CHECK(error_if_any(ctx, rc) == std::nullopt);
+    if (rc == TILEDB_OK) {
+      CHECK(upper_bound == std::vector<uint64_t>{2, 2});
+    }
+    rc = tiledb_fragment_info_get_global_order_upper_bound(
+        ctx, fragment_info, 0, 1, nullptr, &dimensions[0]);
+    CHECK(rc == TILEDB_ERR);
+
+    // second fragment - two tiles
+    rc = tiledb_fragment_info_get_global_order_upper_bound(
+        ctx, fragment_info, 1, 0, nullptr, &dimensions[0]);
+    CHECK(error_if_any(ctx, rc) == std::nullopt);
+    if (rc == TILEDB_OK) {
+      CHECK(upper_bound == std::vector<uint64_t>{2, 2});
+    }
+    rc = tiledb_fragment_info_get_global_order_upper_bound(
+        ctx, fragment_info, 1, 1, nullptr, &dimensions[0]);
+    CHECK(error_if_any(ctx, rc) == std::nullopt);
+    if (rc == TILEDB_OK) {
+      CHECK(upper_bound == std::vector<uint64_t>{8, 8});
+    }
+    rc = tiledb_fragment_info_get_global_order_upper_bound(
+        ctx, fragment_info, 1, 2, nullptr, &dimensions[0]);
+    CHECK(rc == TILEDB_ERR);
+
+    // third fragment - two tiles
+    rc = tiledb_fragment_info_get_global_order_upper_bound(
+        ctx, fragment_info, 2, 0, nullptr, &dimensions[0]);
+    CHECK(error_if_any(ctx, rc) == std::nullopt);
+    if (rc == TILEDB_OK) {
+      CHECK(upper_bound == std::vector<uint64_t>{2, 2});
+    }
+    rc = tiledb_fragment_info_get_global_order_upper_bound(
+        ctx, fragment_info, 2, 1, nullptr, &dimensions[0]);
+    CHECK(error_if_any(ctx, rc) == std::nullopt);
+    if (rc == TILEDB_OK) {
+      CHECK(upper_bound == std::vector<uint64_t>{7, 7});
+    }
+    rc = tiledb_fragment_info_get_global_order_upper_bound(
+        ctx, fragment_info, 2, 2, nullptr, &dimensions[0]);
+    CHECK(rc == TILEDB_ERR);
+  }
+
   // Clean up
   tiledb_fragment_info_free(&fragment_info);
   remove_dir(array_name, ctx, vfs);
@@ -1842,7 +1965,7 @@ TEST_CASE(
       "- Unconsolidated metadata num: 1\n" + "- To vacuum num: 0\n" +
       "- Fragment #1:\n" + "  > URI: " + written_frag_uri + "\n" +
       "  > Schema name: " + schema_name + "\n" + "  > Type: sparse\n" +
-      "  > Non-empty domain: [a, ddd]\n" + "  > Size: 3439\n" +
+      "  > Non-empty domain: [a, ddd]\n" + "  > Size: 3674\n" +
       "  > Cell num: 4\n" + "  > Timestamp range: [1, 1]\n" +
       "  > Format version: " + ver + "\n" +
       "  > Has consolidated metadata: no\n";