diff --git a/format_spec/FORMAT_SPEC.md b/format_spec/FORMAT_SPEC.md index ed952a9dbbf..c7356bfb8df 100644 --- a/format_spec/FORMAT_SPEC.md +++ b/format_spec/FORMAT_SPEC.md @@ -4,7 +4,7 @@ title: Format Specification **Notes:** -* The current TileDB array format version number is **22** (`uint32_t`). +* The current TileDB array format version number is **23** (`uint32_t`). * Other structures might be versioned separately. * Data written by TileDB and referenced in this document is **little-endian** with the following exceptions: diff --git a/format_spec/array_format_history.md b/format_spec/array_format_history.md index 8c3f5512ef9..c1fe2909a89 100644 --- a/format_spec/array_format_history.md +++ b/format_spec/array_format_history.md @@ -4,6 +4,12 @@ title: Array format version history # Array Format Version History +## Version 23 + +Introduced in TileDB 2.30 + +* The _Tile global order min/max_ fields were added to [tile metadata](./fragment.md#tile-mins-maxes). These fields contain the minimum and maximum global order coordinate for each tile in the fragment. This metadata can be used to optimize query execution. + ## Version 22 Introduced in TileDB 2.25 diff --git a/format_spec/fragment.md b/format_spec/fragment.md index 78dd065b0a4..3a9d09f2896 100644 --- a/format_spec/fragment.md +++ b/format_spec/fragment.md @@ -82,6 +82,12 @@ The fragment metadata file has the following on-disk format: | Tile maxes for attribute/dimension 1 | [Tile Mins/Maxes](#tile-mins-maxes) | _New in version 11_ The serialized maxes for attribute/dimension 1 | | … | … | … | | Variable maxes for attribute/dimension N | [Tile Mins/Maxes](#tile-mins-maxes) | _New in version 11_ The serialized maxes for attribute/dimension N | +| Tile global order min coordinates for dimension 1 | [Tile Mins/Maxes](#tile-mins-maxes) | _New in version 23_ For sparse arrays, the serialized value of dimension 1 of the global order minimum coordinate in each tile. | +| … | … | … | +| Variable global order min coordinates for dimension N | [Tile Mins/Maxes](#tile-mins-maxes) | _New in version 23_ For sparse arrays, the serialized value of dimension N of the global order minimum coordinate in each tile. | +| Tile global order max coordinates for dimension 1 | [Tile Mins/Maxes](#tile-mins-maxes) | _New in version 23_ For sparse arrays, the serialized value of dimension 1 of the global order maximum coordinate in each tile. | +| … | … | … | +| Variable global order max coordinates for dimension N | [Tile Mins/Maxes](#tile-mins-maxes) | _New in version 23_ For sparse arrays, the serialized value of dimension N of the global order maximum coordinate in each tile. | | Tile sums for attribute/dimension 1 | [Tile Sums](#tile-sums) | _New in version 11_ The serialized sums for attribute/dimension 1 | | … | … | … | | Variable sums for attribute/dimension N | [Tile Sums](#tile-sums) | _New in version 11_ The serialized sums for attribute/dimension N | @@ -276,6 +282,12 @@ The footer is a simple blob \(i.e., _not a generic tile_\) with the following in | Tile maxes offset for attribute/dimension 1 | `uint64_t` | The offset to the generic tile storing the tile maxes for attribute/dimension 1. | | … | … | … | | Tile maxes offset for attribute/dimension N | `uint64_t` | The offset to the generic tile storing the tile maxes for attribute/dimension N | +| Tile global order min coordinates offset for dimension 1 | `uint64_t` | _New in version 23_ For sparse arrays, the offset to the generic tile storing the tile global order mins for dimension 1 +| … | … | … | +| Tile global order min coordinates offset for dimension N | `uint64_t` | _New in version 23_ For sparse arrays, the offset to the generic tile storing the tile global order mins for dimension N +| Tile global order max coordinates offset for dimension 1 | `uint64_t` | _New in version 23_ For sparse arrays, the offset to the generic tile storing the tile global order maxes for dimension 1 +| … | … | … | +| Tile global order max coordinates offset for dimension N | `uint64_t` | _New in version 23_ For sparse arrays, the offset to the generic tile storing the tile global order maxes for dimension N | Tile sums offset for attribute/dimension 1 | `uint64_t` | The offset to the generic tile storing the tile sums for attribute/dimension 1. | | … | … | … | | Tile sums offset for attribute/dimension N | `uint64_t` | The offset to the generic tile storing the tile sums for attribute/dimension N | diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index f7753d991d0..e7f4dc11fae 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -110,6 +110,7 @@ set(TILEDB_UNIT_TEST_SOURCES src/unit-enumerations.cc src/unit-enum-helpers.cc src/unit-filter-buffer.cc + src/unit-fragment-info-global-order-bounds.cc src/unit-global-order.cc src/unit-ordered-dim-label-reader.cc src/unit-tile-metadata.cc diff --git a/test/src/unit-capi-fragment_info.cc b/test/src/unit-capi-fragment_info.cc index 2c12d6eefe7..887a4174006 100644 --- a/test/src/unit-capi-fragment_info.cc +++ b/test/src/unit-capi-fragment_info.cc @@ -30,6 +30,7 @@ * Tests the C API functions for manipulating fragment information. */ +#include "test/support/src/error_helpers.h" #include "test/support/src/helpers.h" #include "test/support/src/serialization_wrappers.h" #include "tiledb/sm/c_api/tiledb.h" @@ -553,6 +554,22 @@ TEST_CASE( ctx, fragment_info, 1, 0, "d", &mbr[0]); CHECK(rc == TILEDB_ERR); + // Get global order lower bound - should fail since it's a dense array + { + void* dimensions[] = {&mbr[0], &mbr[1]}; + rc = tiledb_fragment_info_get_global_order_lower_bound( + ctx, fragment_info, 0, 0, nullptr, &dimensions[0]); + CHECK(rc == TILEDB_ERR); + } + + // Get global order upper bound - should fail since it's a dense array + { + void* dimensions[] = {&mbr[0], &mbr[1]}; + rc = tiledb_fragment_info_get_global_order_upper_bound( + ctx, fragment_info, 0, 0, nullptr, &dimensions[0]); + CHECK(rc == TILEDB_ERR); + } + // Get version uint32_t version; rc = tiledb_fragment_info_get_version(ctx, fragment_info, 0, &version); @@ -703,13 +720,14 @@ TEST_CASE("C API: Test MBR fragment info", "[capi][fragment_info][mbr]") { // Load fragment info rc = tiledb_fragment_info_load(ctx, fragment_info); CHECK(rc == TILEDB_OK); - tiledb_config_free(&cfg); tiledb_fragment_info_t* deserialized_fragment_info = nullptr; if (serialized_load) { rc = tiledb_fragment_info_alloc( ctx, array_name.c_str(), &deserialized_fragment_info); CHECK(rc == TILEDB_OK); + rc = tiledb_fragment_info_set_config(ctx, deserialized_fragment_info, cfg); + CHECK(rc == TILEDB_OK); tiledb_fragment_info_serialize( ctx, array_name.c_str(), @@ -720,6 +738,8 @@ TEST_CASE("C API: Test MBR fragment info", "[capi][fragment_info][mbr]") { fragment_info = deserialized_fragment_info; } + tiledb_config_free(&cfg); + // Get fragment num uint32_t fragment_num; rc = tiledb_fragment_info_get_fragment_num(ctx, fragment_info, &fragment_num); @@ -731,6 +751,7 @@ TEST_CASE("C API: Test MBR fragment info", "[capi][fragment_info][mbr]") { rc = tiledb_fragment_info_get_mbr_num(ctx, fragment_info, 0, &mbr_num); CHECK(rc == TILEDB_OK); CHECK(mbr_num == 1); + rc = tiledb_fragment_info_get_mbr_num(ctx, fragment_info, 1, &mbr_num); CHECK(rc == TILEDB_OK); CHECK(mbr_num == 2); @@ -753,6 +774,108 @@ TEST_CASE("C API: Test MBR fragment info", "[capi][fragment_info][mbr]") { CHECK(rc == TILEDB_OK); CHECK(mbr == std::vector{7, 8}); + // Get global order lower bounds + { + std::vector lower_bound(2); + void* dimensions[] = {&lower_bound[0], &lower_bound[1]}; + + // first fragment - one tile + rc = tiledb_fragment_info_get_global_order_lower_bound( + ctx, fragment_info, 0, 0, nullptr, &dimensions[0]); + CHECK(error_if_any(ctx, rc) == std::nullopt); + if (rc == TILEDB_OK) { + CHECK(lower_bound == std::vector{1, 1}); + } + rc = tiledb_fragment_info_get_global_order_lower_bound( + ctx, fragment_info, 0, 1, nullptr, &dimensions[0]); + CHECK(rc == TILEDB_ERR); + + // second fragment - two tiles + rc = tiledb_fragment_info_get_global_order_lower_bound( + ctx, fragment_info, 1, 0, nullptr, &dimensions[0]); + CHECK(error_if_any(ctx, rc) == std::nullopt); + if (rc == TILEDB_OK) { + CHECK(lower_bound == std::vector{1, 1}); + } + rc = tiledb_fragment_info_get_global_order_lower_bound( + ctx, fragment_info, 1, 1, nullptr, &dimensions[0]); + CHECK(error_if_any(ctx, rc) == std::nullopt); + if (rc == TILEDB_OK) { + CHECK(lower_bound == std::vector{7, 7}); + } + rc = tiledb_fragment_info_get_global_order_lower_bound( + ctx, fragment_info, 1, 2, nullptr, &dimensions[0]); + CHECK(rc == TILEDB_ERR); + + // third fragment - two tiles + rc = tiledb_fragment_info_get_global_order_lower_bound( + ctx, fragment_info, 2, 0, nullptr, &dimensions[0]); + CHECK(error_if_any(ctx, rc) == std::nullopt); + if (rc == TILEDB_OK) { + CHECK(lower_bound == std::vector{1, 1}); + } + rc = tiledb_fragment_info_get_global_order_lower_bound( + ctx, fragment_info, 2, 1, nullptr, &dimensions[0]); + CHECK(error_if_any(ctx, rc) == std::nullopt); + if (rc == TILEDB_OK) { + CHECK(lower_bound == std::vector{1, 8}); + } + rc = tiledb_fragment_info_get_global_order_lower_bound( + ctx, fragment_info, 2, 2, nullptr, &dimensions[0]); + CHECK(rc == TILEDB_ERR); + } + + // Get global order upper bounds + { + std::vector upper_bound(2); + void* dimensions[] = {&upper_bound[0], &upper_bound[1]}; + + // first fragment - one tile + rc = tiledb_fragment_info_get_global_order_upper_bound( + ctx, fragment_info, 0, 0, nullptr, &dimensions[0]); + CHECK(error_if_any(ctx, rc) == std::nullopt); + if (rc == TILEDB_OK) { + CHECK(upper_bound == std::vector{2, 2}); + } + rc = tiledb_fragment_info_get_global_order_upper_bound( + ctx, fragment_info, 0, 1, nullptr, &dimensions[0]); + CHECK(rc == TILEDB_ERR); + + // second fragment - two tiles + rc = tiledb_fragment_info_get_global_order_upper_bound( + ctx, fragment_info, 1, 0, nullptr, &dimensions[0]); + CHECK(error_if_any(ctx, rc) == std::nullopt); + if (rc == TILEDB_OK) { + CHECK(upper_bound == std::vector{2, 2}); + } + rc = tiledb_fragment_info_get_global_order_upper_bound( + ctx, fragment_info, 1, 1, nullptr, &dimensions[0]); + CHECK(error_if_any(ctx, rc) == std::nullopt); + if (rc == TILEDB_OK) { + CHECK(upper_bound == std::vector{8, 8}); + } + rc = tiledb_fragment_info_get_global_order_upper_bound( + ctx, fragment_info, 1, 2, nullptr, &dimensions[0]); + CHECK(rc == TILEDB_ERR); + + // third fragment - two tiles + rc = tiledb_fragment_info_get_global_order_upper_bound( + ctx, fragment_info, 2, 0, nullptr, &dimensions[0]); + CHECK(error_if_any(ctx, rc) == std::nullopt); + if (rc == TILEDB_OK) { + CHECK(upper_bound == std::vector{2, 2}); + } + rc = tiledb_fragment_info_get_global_order_upper_bound( + ctx, fragment_info, 2, 1, nullptr, &dimensions[0]); + CHECK(error_if_any(ctx, rc) == std::nullopt); + if (rc == TILEDB_OK) { + CHECK(upper_bound == std::vector{7, 7}); + } + rc = tiledb_fragment_info_get_global_order_upper_bound( + ctx, fragment_info, 2, 2, nullptr, &dimensions[0]); + CHECK(rc == TILEDB_ERR); + } + // Clean up tiledb_fragment_info_free(&fragment_info); remove_dir(array_name, ctx, vfs); @@ -1842,7 +1965,7 @@ TEST_CASE( "- Unconsolidated metadata num: 1\n" + "- To vacuum num: 0\n" + "- Fragment #1:\n" + " > URI: " + written_frag_uri + "\n" + " > Schema name: " + schema_name + "\n" + " > Type: sparse\n" + - " > Non-empty domain: [a, ddd]\n" + " > Size: 3439\n" + + " > Non-empty domain: [a, ddd]\n" + " > Size: 3674\n" + " > Cell num: 4\n" + " > Timestamp range: [1, 1]\n" + " > Format version: " + ver + "\n" + " > Has consolidated metadata: no\n"; diff --git a/test/src/unit-fragment-info-global-order-bounds.cc b/test/src/unit-fragment-info-global-order-bounds.cc new file mode 100644 index 00000000000..abf1a08b818 --- /dev/null +++ b/test/src/unit-fragment-info-global-order-bounds.cc @@ -0,0 +1,1731 @@ +/** + * @file unit-sparse-global-order-reader.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2025 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests populating fragment metadata with the global order lower/upper bound + * coordinates per tile. + */ + +#include "tiledb/api/c_api/array_schema/array_schema_api_internal.h" +#include "tiledb/sm/misc/comparators.h" + +#include +#include +#include +#include + +using namespace tiledb; +using namespace tiledb::test; + +using Fragment1DFixed = templates::Fragment1D; +using Fragment2DFixed = templates::Fragment2D; + +using Fragment1DVar = + templates::Fragment1D; + +using FragmentVcf2025 = templates::Fragment3D< + templates::StringDimensionCoordType, + uint32_t, + templates::StringDimensionCoordType>; + +void showValue(const Fragment1DFixed& value, std::ostream& os) { + rc::showFragment(value, os); +} + +void showValue(const Fragment1DVar& value, std::ostream& os) { + rc::showFragment(value, os); +} + +void showValue(const Fragment2DFixed& value, std::ostream& os) { + rc::showFragment(value, os); +} + +namespace rc::detail { + +template +struct ShowDefault { + static void show(const Fragment1DFixed& value, std::ostream& os) { + rc::showFragment(value, os); + } +}; + +template +struct ShowDefault { + static void show(const Fragment1DVar& value, std::ostream& os) { + rc::showFragment(value, os); + } +}; + +} // namespace rc::detail + +/** + * @return another fragment containing the contents of the argument sorted in + * global order based on `array.schema()`. + */ +template +static F make_global_order( + const Array& array, const F& fragment, sm::Layout layout) { + if (layout == sm::Layout::GLOBAL_ORDER) { + return fragment; + } + + std::vector idxs(fragment.size(), 0); + std::iota(idxs.begin(), idxs.end(), 0); + + // sort in global order + auto array_schema = array.schema().ptr()->array_schema(); + sm::GlobalCellCmp globalcmp(array_schema->domain()); + + auto icmp = [&](uint64_t ia, uint64_t ib) -> bool { + return std::apply( + [&globalcmp, ia, ib]( + const templates::query_buffers&... dims) { + const auto l = std::make_tuple(dims[ia]...); + const auto r = std::make_tuple(dims[ib]...); + return globalcmp( + templates::global_cell_cmp_std_tuple(l), + templates::global_cell_cmp_std_tuple(r)); + }, + fragment.dimensions()); + }; + std::sort(idxs.begin(), idxs.end(), icmp); + + F sorted = fragment; + sorted.dimensions() = + stdx::select(stdx::reference_tuple(sorted.dimensions()), std::span(idxs)); + if constexpr (!std::is_same_v>) { + sorted.attributes() = stdx::select( + stdx::reference_tuple(sorted.attributes()), std::span(idxs)); + } + + return sorted; +} + +template +using DimensionTuple = + stdx::decay_tuple().dimensions())>; + +template +T to_owned(T value) { + return value; +} + +template +std::vector to_owned(std::span value) { + return std::vector(value.begin(), value.end()); +} + +template +auto tuple_index(const T& tuple, uint64_t idx) { + return std::apply( + [&](const auto&... field) { + return std::make_tuple(to_owned(field[idx])...); + }, + tuple); +} + +template +using CoordsTuple = decltype(tuple_index( + std::declval().dimensions(), std::declval())); + +template +using Bounds = std::pair, CoordsTuple>; + +template +using FragmentBounds = std::vector>; + +template +using ArrayBounds = std::vector>; + +/** + * @return the lower and upper bounds of tile `(f, t)` in the fragment info + */ +template +static Bounds global_order_bounds( + const FragmentInfo& finfo, uint64_t fragment, uint64_t tile) { + DimensionTuple lb, ub; + + auto bound_vec_to_tuple = [](DimensionTuple& out, + std::vector> bounds) { + auto handle_field = [&]( + templates::query_buffers& qb, uint64_t d) { + static_assert( + stdx::is_fundamental || + std::is_same_v); + if constexpr (stdx::is_fundamental) { + qb.resize(1); + qb[0] = *reinterpret_cast(bounds[d].data()); + } else { + qb.values_.resize(bounds[d].size()); + memcpy(qb.values_.data(), bounds[d].data(), bounds[d].size()); + qb.offsets_ = {0}; + } + }; + uint64_t d = 0; + std::apply( + [&](templates::query_buffers&... qb) { + (handle_field(qb, d++), ...); + }, + out); + }; + + auto lbvec = finfo.global_order_lower_bound(fragment, tile); + bound_vec_to_tuple(lb, lbvec); + + auto ubvec = finfo.global_order_upper_bound(fragment, tile); + bound_vec_to_tuple(ub, ubvec); + + return std::make_pair(tuple_index(lb, 0), tuple_index(ub, 0)); +} + +/** + * @return the global order bounds of all tiles in all fragments; [f][t] + * contains the bounds of the t'th tile of fragment f + */ +template +std::vector>> get_all_bounds( + const Context& ctx, const std::string& array_uri) { + FragmentInfo finfo(ctx, array_uri); + finfo.load(); + + std::vector>> bounds; + bounds.reserve(finfo.fragment_num()); + + for (uint64_t f = 0; f < finfo.fragment_num(); f++) { + const uint64_t num_tiles = finfo.mbr_num(f); + + std::vector> this_fragment_bounds; + this_fragment_bounds.reserve(num_tiles); + for (uint64_t t = 0; t < num_tiles; t++) { + this_fragment_bounds.push_back(global_order_bounds(finfo, f, t)); + } + + bounds.push_back(this_fragment_bounds); + } + + return bounds; +} + +/** + * Asserts that when a set of fragments are written, the fragment metadata + * accurately reflects the expected global order bounds of the input. + * + * "Accurately reflects" means that: + * 1) the lower bound is indeed the first coordinate in global order in the + * fragment 2) the upper bound is indeed the last coordinate in global order in + * the fragment + * + * @return the global order bounds for each tile per fragment + */ +template +std::vector>> assert_written_bounds( + const Context& ctx, + const std::string& array_uri, + const std::vector& fragments, + sm::Layout layout = sm::Layout::GLOBAL_ORDER) { + // write each fragment + { + Array forwrite(ctx, array_uri, TILEDB_WRITE); + for (const auto& fragment : fragments) { + templates::query::write_fragment( + fragment, forwrite, static_cast(layout)); + } + } + + // check bounds + std::vector>> bounds = + get_all_bounds(ctx, array_uri); + + Array forread(ctx, array_uri, TILEDB_READ); + const uint64_t tile_stride = forread.schema().capacity(); + + ASSERTER(bounds.size() == fragments.size()); + for (size_t f = 0; f < fragments.size(); f++) { + const auto fragment = make_global_order(forread, fragments[f], layout); + + const uint64_t num_tiles = bounds[f].size(); + ASSERTER(num_tiles == (fragment.size() + tile_stride - 1) / tile_stride); + + for (size_t t = 0; t < num_tiles; t++) { + const uint64_t lbi = t * tile_stride; + const uint64_t ubi = + std::min((t + 1) * tile_stride, fragment.size()) - 1; + + const auto lbexpect = tuple_index(fragment.dimensions(), lbi); + const auto ubexpect = tuple_index(fragment.dimensions(), ubi); + + const auto [lbactual, ubactual] = bounds[f][t]; + ASSERTER(lbexpect == lbactual); + ASSERTER(ubexpect == ubactual); + } + } + + return bounds; +} + +TEST_CASE( + "Fragment metadata global order bounds: 1D fixed", + "[fragment_info][global-order]") { + VFSTestSetup vfs_test_setup; + const auto array_uri = vfs_test_setup.array_uri( + "fragment_metadata_global_order_bounds_1d_fixed"); + + const templates::Dimension dimension( + templates::Domain(0, 1024 * 8), 16); + + const bool allow_dups = GENERATE(true, false); + + templates::ddl::create_array( + array_uri, + vfs_test_setup.ctx(), + std::tuple&>{dimension}, + std::vector>{}, + TILEDB_ROW_MAJOR, + TILEDB_ROW_MAJOR, + 8, + allow_dups); + + DeleteArrayGuard delarray( + vfs_test_setup.ctx().ptr().get(), array_uri.c_str()); + + using Fragment = templates::Fragment1D; + + auto make_expect = + [](std::initializer_list> tile_bounds) { + std::vector, std::tuple>> + out_bounds; + for (const auto& tile : tile_bounds) { + out_bounds.push_back(std::make_pair( + std::make_tuple(tile.first), std::make_tuple(tile.second))); + } + return out_bounds; + }; + + SECTION("Minimum write") { + Fragment f; + f.resize(1); + f.dimension()[0] = 1; + + std::vector>> fragment_bounds; + SECTION("Global Order") { + fragment_bounds = + assert_written_bounds( + vfs_test_setup.ctx(), array_uri, std::vector{f}); + } + + SECTION("Unordered") { + fragment_bounds = + assert_written_bounds( + vfs_test_setup.ctx(), + array_uri, + std::vector{f}, + sm::Layout::UNORDERED); + } + REQUIRE(fragment_bounds.size() == 1); + CHECK( + fragment_bounds[0] == std::vector>{std::make_pair( + std::make_tuple(1), std::make_tuple(1))}); + } + + SECTION("Ascending fragment") { + Fragment f; + f.resize(64); + + const auto expect = make_expect({ + {1, 8}, + {9, 16}, + {17, 24}, + {25, 32}, + {33, 40}, + {41, 48}, + {49, 56}, + {57, 64}, + }); + + SECTION("Global Order") { + std::iota(f.dimension().begin(), f.dimension().end(), 1); + const auto fragment_bounds = + assert_written_bounds( + vfs_test_setup.ctx(), array_uri, std::vector{f}); + REQUIRE(fragment_bounds.size() == 1); + CHECK(fragment_bounds[0] == expect); + } + + SECTION("Unordered") { + for (uint64_t i = 0; i < f.size(); i++) { + f.dimension()[i] = f.size() - i; + } + + const auto fragment_bounds = + assert_written_bounds( + vfs_test_setup.ctx(), + array_uri, + std::vector{f}, + sm::Layout::UNORDERED); + REQUIRE(fragment_bounds.size() == 1); + CHECK(fragment_bounds[0] == expect); + } + } + + if (allow_dups) { + SECTION("Duplicates") { + const auto expect = make_expect({{0, 0}, {1, 1}}); + + SECTION("Global Order") { + Fragment f; + f.dimension() = {0, 0, 0, 0, 0, 0, 0, 0, 1}; + + const auto fragment_bounds = + assert_written_bounds( + vfs_test_setup.ctx(), + array_uri, + std::vector{f}, + sm::Layout::GLOBAL_ORDER); + REQUIRE(fragment_bounds.size() == 1); + CHECK(fragment_bounds[0] == expect); + } + + SECTION("Unordered") { + Fragment f; + f.dimension() = {0, 0, 0, 1, 0, 0, 0, 0, 0}; + + const auto fragment_bounds = + assert_written_bounds( + vfs_test_setup.ctx(), + array_uri, + std::vector{f}, + sm::Layout::UNORDERED); + REQUIRE(fragment_bounds.size() == 1); + CHECK(fragment_bounds[0] == expect); + } + } + } +} + +TEST_CASE( + "Fragment metadata global order bounds: 1D fixed rapidcheck", + "[fragment_info][global-order][rapidcheck]") { + VFSTestSetup vfs_test_setup; + const auto array_uri = vfs_test_setup.array_uri( + "fragment_metadata_global_order_bounds_1d_fixed_rapidcheck"); + + static constexpr uint64_t LB = 0; + static constexpr uint64_t UB = 1024 * 8; + const templates::Domain domain(LB, UB); + const templates::Dimension dimension(domain, 16); + + Context ctx = vfs_test_setup.ctx(); + + auto temp_array = [&](bool allow_dups) { + templates::ddl::create_array( + array_uri, + ctx, + std::tuple&>{dimension}, + std::vector>{}, + TILEDB_ROW_MAJOR, + TILEDB_ROW_MAJOR, + 8, + allow_dups); + + return DeleteArrayGuard(ctx.ptr().get(), array_uri.c_str()); + }; + + rc::prop("global order", [&](bool allow_dups) { + auto fragments = *rc::gen::container>( + rc::make_fragment_1d(allow_dups, domain)); + auto arrayguard = temp_array(allow_dups); + Array forread(ctx, array_uri, TILEDB_READ); + std::vector global_order_fragments; + for (const auto& fragment : fragments) { + global_order_fragments.push_back(make_global_order( + forread, fragment, sm::Layout::UNORDERED)); + } + + assert_written_bounds( + vfs_test_setup.ctx(), + array_uri, + global_order_fragments, + sm::Layout::GLOBAL_ORDER); + }); + + rc::prop("unordered", [&]() { + const bool allow_dups = false; // FIXME: not working correctly + auto fragments = *rc::gen::container>( + rc::make_fragment_1d(allow_dups, domain)); + auto arrayguard = temp_array(allow_dups); + Array forread(ctx, array_uri, TILEDB_READ); + + assert_written_bounds( + vfs_test_setup.ctx(), array_uri, fragments, sm::Layout::UNORDERED); + }); +} + +TEST_CASE( + "Fragment metadata global order bounds: 2D fixed", + "[fragment_info][global-order]") { + VFSTestSetup vfs_test_setup; + const auto array_uri = vfs_test_setup.array_uri( + "fragment_metadata_global_order_bounds_2d_fixed"); + + const bool allow_dups = false; + + const templates::Dimension d1( + templates::Domain(-256, 256), 4); + const templates::Dimension d2( + templates::Domain(-256, 256), 4); + + Context ctx = vfs_test_setup.ctx(); + + templates::ddl::create_array( + array_uri, + ctx, + std::tie(d1, d2), + std::vector>{}, + TILEDB_ROW_MAJOR, + TILEDB_ROW_MAJOR, + 8, + allow_dups); + + DeleteArrayGuard delarray(ctx.ptr().get(), array_uri.c_str()); + + using Fragment = templates::Fragment2D; + using TileBounds = + std::pair, std::tuple>; + + Fragment minimum; + { + minimum.resize(1); + minimum.d1()[0] = 0; + minimum.d2()[0] = 0; + } + + constexpr size_t row_num_cells = 64, col_num_cells = 64, + square_num_cells = 64; + + Fragment row, col, square, square_offset; + + row.resize(row_num_cells); + col.resize(col_num_cells); + square.resize(square_num_cells); + square_offset.resize(square_num_cells); + + const sm::Layout layout = + GENERATE(sm::Layout::UNORDERED, sm::Layout::GLOBAL_ORDER); + + for (uint64_t i = 0; i < row_num_cells; i++) { + row.d1()[i] = 0; + row.d2()[i] = i; + } + + for (uint64_t i = 0; i < col_num_cells; i++) { + col.d1()[i] = i; + col.d2()[i] = 0; + } + + const uint64_t square_row_length = std::sqrt(square_num_cells); + for (uint64_t i = 0; i < square_num_cells; i++) { + square.d1()[i] = i / square_row_length; + square.d2()[i] = i % square_row_length; + } + for (uint64_t i = 0; i < square_num_cells; i++) { + square_offset.d1()[i] = 2 + (i / square_row_length); + square_offset.d2()[i] = 2 + (i % square_row_length); + } + + if (layout == sm::Layout::GLOBAL_ORDER) { + Array forread(ctx, array_uri, TILEDB_READ); + + // row, col are in global order already + square = make_global_order(forread, square, tiledb::sm::Layout::UNORDERED); + REQUIRE( + square.d1() == std::vector{ + 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, + 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, + 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, + 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, + }); + REQUIRE( + square.d2() == std::vector{ + 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, + 4, 5, 6, 7, 4, 5, 6, 7, 4, 5, 6, 7, 4, 5, 6, 7, + 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, + 4, 5, 6, 7, 4, 5, 6, 7, 4, 5, 6, 7, 4, 5, 6, 7, + }); + + square_offset = make_global_order( + forread, square_offset, tiledb::sm::Layout::UNORDERED); + REQUIRE( + square_offset.d1().values_ == + std::vector{2, 2, 3, 3, 2, 2, 2, 2, 3, 3, 3, 3, 2, 2, 3, 3, + 4, 4, 5, 5, 6, 6, 7, 7, 4, 4, 4, 4, 5, 5, 5, 5, + 6, 6, 6, 6, 7, 7, 7, 7, 4, 4, 5, 5, 6, 6, 7, 7, + 8, 8, 9, 9, 8, 8, 8, 8, 9, 9, 9, 9, 8, 8, 9, 9}); + REQUIRE( + square_offset.d2().values_ == + std::vector{2, 3, 2, 3, 4, 5, 6, 7, 4, 5, 6, 7, 8, 9, 8, 9, + 2, 3, 2, 3, 2, 3, 2, 3, 4, 5, 6, 7, 4, 5, 6, 7, + 4, 5, 6, 7, 4, 5, 6, 7, 8, 9, 8, 9, 8, 9, 8, 9, + 2, 3, 2, 3, 4, 5, 6, 7, 4, 5, 6, 7, 8, 9, 8, 9}); + } + + const std::vector expect_row_bounds = { + {{0, 0}, {0, 7}}, + {{0, 8}, {0, 15}}, + {{0, 16}, {0, 23}}, + {{0, 24}, {0, 31}}, + {{0, 32}, {0, 39}}, + {{0, 40}, {0, 47}}, + {{0, 48}, {0, 55}}, + {{0, 56}, {0, 63}}}; + + const std::vector expect_col_bounds = { + {{0, 0}, {7, 0}}, + {{8, 0}, {15, 0}}, + {{16, 0}, {23, 0}}, + {{24, 0}, {31, 0}}, + {{32, 0}, {39, 0}}, + {{40, 0}, {47, 0}}, + {{48, 0}, {55, 0}}, + {{56, 0}, {63, 0}}}; + + const std::vector expect_square_bounds = { + {{0, 0}, {1, 3}}, + {{2, 0}, {3, 3}}, + {{0, 4}, {1, 7}}, + {{2, 4}, {3, 7}}, + {{4, 0}, {5, 3}}, + {{6, 0}, {7, 3}}, + {{4, 4}, {5, 7}}, + {{6, 4}, {7, 7}}, + }; + + const std::vector expect_square_offset_bounds = { + {{2, 2}, {2, 7}}, + {{3, 4}, {3, 9}}, + {{4, 2}, {7, 3}}, + {{4, 4}, {5, 7}}, + {{6, 4}, {7, 7}}, + {{4, 8}, {7, 9}}, + {{8, 2}, {8, 7}}, + {{9, 4}, {9, 9}}}; + + SECTION("Minimum write") { + Fragment f; + f.resize(1); + f.d1()[0] = 0; + f.d2()[0] = 0; + + std::vector>> fragment_bounds; + SECTION("Global Order") { + fragment_bounds = + assert_written_bounds( + ctx, array_uri, std::vector{f}); + } + + SECTION("Unordered") { + fragment_bounds = + assert_written_bounds( + ctx, array_uri, std::vector{f}, sm::Layout::UNORDERED); + } + + std::vector expect_bounds = {{{0, 0}, {0, 0}}}; + REQUIRE(fragment_bounds.size() == 1); + CHECK(fragment_bounds[0] == expect_bounds); + } + + DYNAMIC_SECTION("Row (layout = " + sm::layout_str(layout) + ")") { + const auto fragment_bounds = + assert_written_bounds( + ctx, array_uri, std::vector{row}, layout); + REQUIRE(fragment_bounds.size() == 1); + CHECK(fragment_bounds[0] == expect_row_bounds); + } + + DYNAMIC_SECTION("Column (layout = " + sm::layout_str(layout) + ")") { + const auto fragment_bounds = + assert_written_bounds( + ctx, array_uri, std::vector{col}, layout); + REQUIRE(fragment_bounds.size() == 1); + CHECK(fragment_bounds[0] == expect_col_bounds); + } + + DYNAMIC_SECTION("Square (layout = " + sm::layout_str(layout) + ")") { + const auto fragment_bounds = + assert_written_bounds( + ctx, array_uri, std::vector{square}, layout); + REQUIRE(fragment_bounds.size() == 1); + CHECK(fragment_bounds[0] == expect_square_bounds); + } + + DYNAMIC_SECTION("Square offset (layout = " + sm::layout_str(layout) + ")") { + const auto fragment_bounds = + assert_written_bounds( + ctx, array_uri, std::vector{square_offset}, layout); + REQUIRE(fragment_bounds.size() == 1); + CHECK(fragment_bounds[0] == expect_square_offset_bounds); + } + + DYNAMIC_SECTION("Multi-fragment (layout = " + sm::layout_str(layout) + ")") { + const std::vector fragments = {col, square_offset, row, square}; + const auto fragment_bounds = + assert_written_bounds( + ctx, array_uri, fragments, layout); + CHECK(fragment_bounds.size() >= 1); + if (fragment_bounds.size() >= 1) { + CHECK(fragment_bounds[0] == expect_col_bounds); + } + CHECK(fragment_bounds.size() >= 2); + if (fragment_bounds.size() >= 2) { + CHECK(fragment_bounds[1] == expect_square_offset_bounds); + } + CHECK(fragment_bounds.size() >= 3); + if (fragment_bounds.size() >= 3) { + CHECK(fragment_bounds[2] == expect_row_bounds); + } + CHECK(fragment_bounds.size() >= 4); + if (fragment_bounds.size() >= 4) { + CHECK(fragment_bounds[3] == expect_square_bounds); + } + REQUIRE(fragment_bounds.size() == 4); + } +} + +TEST_CASE( + "Fragment metadata global order bounds: 2D fixed rapidcheck", + "[fragment_info][global-order][rapidcheck]") { + VFSTestSetup vfs_test_setup; + const auto array_uri = vfs_test_setup.array_uri( + "fragment_metadata_global_order_bounds_1d_fixed_rapidcheck"); + + static constexpr int32_t LB = -256; + static constexpr int32_t UB = 256; + + const templates::Domain domain(LB, UB); + const templates::Dimension d1(domain, 4); + const templates::Dimension d2(domain, 4); + + Context ctx = vfs_test_setup.ctx(); + + auto temp_array = [&](bool allow_dups) { + templates::ddl::create_array( + array_uri, + ctx, + std::tuple< + const templates::Dimension&, + const templates::Dimension&>{d1, d2}, + std::vector>{}, + TILEDB_ROW_MAJOR, + TILEDB_ROW_MAJOR, + 8, + allow_dups); + + return DeleteArrayGuard(ctx.ptr().get(), array_uri.c_str()); + }; + + rc::prop("global order", [&](bool allow_dups) { + auto fragments = *rc::gen::container>( + rc::make_fragment_2d(allow_dups, domain, domain)); + auto arrayguard = temp_array(allow_dups); + Array forread(ctx, array_uri, TILEDB_READ); + std::vector global_order_fragments; + for (const auto& fragment : fragments) { + global_order_fragments.push_back(make_global_order( + forread, fragment, sm::Layout::UNORDERED)); + } + + assert_written_bounds( + vfs_test_setup.ctx(), + array_uri, + global_order_fragments, + sm::Layout::GLOBAL_ORDER); + }); + + rc::prop("unordered", [&]() { + const bool allow_dups = false; // FIXME: not working correctly + auto fragments = *rc::gen::container>( + rc::make_fragment_2d(allow_dups, domain, domain)); + auto arrayguard = temp_array(allow_dups); + Array forread(ctx, array_uri, TILEDB_READ); + + assert_written_bounds( + vfs_test_setup.ctx(), array_uri, fragments, sm::Layout::UNORDERED); + }); +} + +TEST_CASE( + "Fragment metadata global order bounds: 1D var", + "[fragment_info][global-order]") { + VFSTestSetup vfs_test_setup; + const auto array_uri = + vfs_test_setup.array_uri("fragment_metadata_global_order_bounds_1d_var"); + + const bool allow_dups = GENERATE(true, false); + + const templates::Dimension dimension; + + Context ctx = vfs_test_setup.ctx(); + + templates::ddl::create_array( + array_uri, + ctx, + std::tuple&>{ + dimension}, + std::vector>{}, + TILEDB_ROW_MAJOR, + TILEDB_ROW_MAJOR, + 32, + allow_dups); + + DeleteArrayGuard delarray(ctx.ptr().get(), array_uri.c_str()); + + using Fragment = Fragment1DVar; + + SECTION("Minimum write") { + std::string svalue = GENERATE("foo", "", "long-ish string"); + const templates::StringDimensionCoordType value( + svalue.begin(), svalue.end()); + + Fragment f; + f.dimension().push_back(value); + + std::vector>> fragment_bounds; + SECTION("Global Order") { + fragment_bounds = + assert_written_bounds( + ctx, array_uri, std::vector{f}); + } + + SECTION("Unordered") { + fragment_bounds = + assert_written_bounds( + ctx, array_uri, std::vector{f}, sm::Layout::UNORDERED); + } + REQUIRE(fragment_bounds.size() == 1); + + CHECK( + fragment_bounds[0] == + std::vector>{ + std::make_pair(std::make_tuple(value), std::make_tuple(value))}); + } + + SECTION("Single fragment") { + const std::vector words = { + "foo", "bar", "baz", "quux", "corge", "grault", "gub"}; + + Fragment f; + for (const auto& s1 : words) { + for (const auto& s2 : words) { + for (const auto& s3 : words) { + templates::StringDimensionCoordType coord(s1.begin(), s1.end()); + coord.insert(coord.end(), s2.begin(), s2.end()); + coord.insert(coord.end(), s3.begin(), s3.end()); + + f.dimension().push_back(coord); + } + } + } + + const sm::Layout layout = + GENERATE(sm::Layout::UNORDERED, sm::Layout::GLOBAL_ORDER); + + if (layout == sm::Layout::GLOBAL_ORDER) { + Array forread(ctx, array_uri, TILEDB_READ); + f = make_global_order(forread, f, tiledb::sm::Layout::UNORDERED); + } + + DYNAMIC_SECTION( + "allow_dups = " + std::to_string(allow_dups) + + ", layout = " + sm::layout_str(layout)) { + const auto fragment_bounds = + assert_written_bounds( + ctx, array_uri, std::vector{f}, layout); + REQUIRE(fragment_bounds.size() == 1); + CHECK(fragment_bounds[0].size() == 11); + + auto lbstr = [](const Bounds& bound) { + const auto& value = std::get<0>(std::get<0>(bound)); + return std::string(value.begin(), value.end()); + }; + auto ubstr = [](const Bounds& bound) { + const auto& value = std::get<0>(std::get<1>(bound)); + return std::string(value.begin(), value.end()); + }; + + CHECK(fragment_bounds[0].size() == 11); + if (fragment_bounds[0].size() > 0) { + CHECK(lbstr(fragment_bounds[0][0]) == "barbarbar"); + CHECK(ubstr(fragment_bounds[0][0]) == "bargraultfoo"); + } + if (fragment_bounds[0].size() > 1) { + CHECK(lbstr(fragment_bounds[0][1]) == "bargraultgrault"); + CHECK(ubstr(fragment_bounds[0][1]) == "bazcorgebar"); + } + if (fragment_bounds[0].size() > 2) { + CHECK(lbstr(fragment_bounds[0][2]) == "bazcorgebaz"); + CHECK(ubstr(fragment_bounds[0][2]) == "bazquuxgrault"); + } + if (fragment_bounds[0].size() > 3) { + CHECK(lbstr(fragment_bounds[0][3]) == "bazquuxgub"); + CHECK(ubstr(fragment_bounds[0][3]) == "corgegraultbaz"); + } + if (fragment_bounds[0].size() > 4) { + CHECK(lbstr(fragment_bounds[0][4]) == "corgegraultcorge"); + CHECK(ubstr(fragment_bounds[0][4]) == "foobazgub"); + } + if (fragment_bounds[0].size() > 5) { + CHECK(lbstr(fragment_bounds[0][5]) == "foobazquux"); + CHECK(ubstr(fragment_bounds[0][5]) == "fooquuxcorge"); + } + if (fragment_bounds[0].size() > 6) { + CHECK(lbstr(fragment_bounds[0][6]) == "fooquuxfoo"); + CHECK(ubstr(fragment_bounds[0][6]) == "graultfooquux"); + } + if (fragment_bounds[0].size() > 7) { + CHECK(lbstr(fragment_bounds[0][7]) == "graultgraultbar"); + CHECK(ubstr(fragment_bounds[0][7]) == "gubbazfoo"); + } + if (fragment_bounds[0].size() > 8) { + CHECK(lbstr(fragment_bounds[0][8]) == "gubbazgrault"); + CHECK(ubstr(fragment_bounds[0][8]) == "gubquuxbar"); + } + if (fragment_bounds[0].size() > 9) { + CHECK(lbstr(fragment_bounds[0][9]) == "gubquuxbaz"); + CHECK(ubstr(fragment_bounds[0][9]) == "quuxfoograult"); + } + if (fragment_bounds[0].size() > 10) { + CHECK(lbstr(fragment_bounds[0][10]) == "quuxfoogub"); + CHECK(ubstr(fragment_bounds[0][10]) == "quuxquuxquux"); + } + } + } +} + +TEST_CASE( + "Fragment metadata global order bounds: 1D var rapidcheck", + "[fragment_info][global-order][rapidcheck]") { + VFSTestSetup vfs_test_setup; + const auto array_uri = vfs_test_setup.array_uri( + "fragment_metadata_global_order_bounds_1d_var_rapidcheck"); + + const templates::Dimension dimension; + + Context ctx = vfs_test_setup.ctx(); + + auto temp_array = [&](bool allow_dups) { + templates::ddl::create_array( + array_uri, + ctx, + std::tuple&>{ + dimension}, + std::vector>{}, + TILEDB_ROW_MAJOR, + TILEDB_ROW_MAJOR, + 8, + allow_dups); + + return DeleteArrayGuard(ctx.ptr().get(), array_uri.c_str()); + }; + + using F = Fragment1DVar; + + auto instance = [&]( + bool allow_dups, const std::vector& fragments) { + auto arrayguard = temp_array(allow_dups); + Array forread(ctx, array_uri, TILEDB_READ); + std::vector global_order_fragments; + for (const auto& fragment : fragments) { + global_order_fragments.push_back( + make_global_order(forread, fragment, sm::Layout::UNORDERED)); + } + + assert_written_bounds( + vfs_test_setup.ctx(), + array_uri, + global_order_fragments, + sm::Layout::GLOBAL_ORDER); + }; + + rc::prop("1D var rapidcheck", [&](bool allow_dups) { + auto fragments = *rc::gen::container>( + rc::make_fragment_1d(allow_dups)); + + instance(allow_dups, fragments); + }); + + SECTION("Shrinking") { + F f; + f.dimension().push_back(templates::StringDimensionCoordType{'a'}); + f.dimension().push_back(templates::StringDimensionCoordType{'b'}); + f.dimension().push_back(templates::StringDimensionCoordType{'c'}); + f.dimension().push_back(templates::StringDimensionCoordType{'w'}); + f.dimension().push_back(templates::StringDimensionCoordType{'n'}); + f.dimension().push_back(templates::StringDimensionCoordType{'a', 'a'}); + f.dimension().push_back(templates::StringDimensionCoordType{'d'}); + f.dimension().push_back(templates::StringDimensionCoordType{'g'}); + f.dimension().push_back(templates::StringDimensionCoordType{'v'}); + + instance.operator()(false, std::vector{f}); + } +} + +/** + * Rapidcheck bounds test using the VCF 2025 data model + * (3D sparse array with chromosome/position/sample dimensions) + */ +TEST_CASE( + "Fragment metadata global order bounds: 3D vcf rapidcheck", + "[fragment_info][global-order][rapidcheck]") { + VFSTestSetup vfs_test_setup; + const auto array_uri = vfs_test_setup.array_uri( + "fragment_metadata_global_order_bounds_3d_vcf_rapidcheck"); + + const templates::Domain domain_sample(0, 10000); + + const templates::Dimension d_chromosome; + const templates::Dimension d_position(domain_sample, 32); + const templates::Dimension d_sample; + + Context ctx = vfs_test_setup.ctx(); + + auto temp_array = [&](bool allow_dups) { + templates::ddl::create_array< + Datatype::STRING_ASCII, + Datatype::UINT32, + Datatype::STRING_ASCII>( + array_uri, + ctx, + std::tie(d_chromosome, d_position, d_sample), + std::vector>{}, + TILEDB_ROW_MAJOR, + TILEDB_ROW_MAJOR, + 8, + allow_dups); + + return DeleteArrayGuard(ctx.ptr().get(), array_uri.c_str()); + }; + + using F = FragmentVcf2025; + + auto instance = [&]( + bool allow_dups, const std::vector& fragments) { + auto arrayguard = temp_array(allow_dups); + Array forread(ctx, array_uri, TILEDB_READ); + std::vector global_order_fragments; + for (const auto& fragment : fragments) { + global_order_fragments.push_back( + make_global_order(forread, fragment, sm::Layout::UNORDERED)); + } + + assert_written_bounds( + vfs_test_setup.ctx(), + array_uri, + global_order_fragments, + sm::Layout::GLOBAL_ORDER); + }; + + rc::prop("3D vcf2025 rapidcheck", [&](bool allow_dups) { + auto fragments = *rc::gen::container>( + rc::make_fragment_3d< + templates::StringDimensionCoordType, + uint32_t, + templates::StringDimensionCoordType>( + allow_dups, std::nullopt, domain_sample, std::nullopt)); + + instance(allow_dups, fragments); + }); +} + +template +std::vector>> consolidate_n_wise( + const Context& ctx, const std::string& uri, uint64_t fan_in) { + // step 0: consolidation config + // NB: this ideally would not be needed but in debug builds + // a huge amount of memory is allocated and initialized which is very slow + Config cfg; + cfg["sm.mem.total_budget"] = std::to_string(128 * 1024 * 1024); + + // step 1: n-wise consolidate + std::vector s_fragment_uris; + { + FragmentInfo fi(ctx, uri); + fi.load(); + + for (uint32_t f = 0; f < fi.fragment_num(); f++) { + s_fragment_uris.push_back(fi.fragment_uri(f)); + } + } + for (uint32_t f = 0; f < s_fragment_uris.size(); f += fan_in) { + std::vector fragment_uris; + for (uint64_t ff = f; + ff < std::min(f + fan_in, s_fragment_uris.size()); + ff++) { + fragment_uris.push_back(s_fragment_uris[ff].c_str()); + } + + Array::consolidate( + ctx, uri, fragment_uris.data(), fragment_uris.size(), &cfg); + } + + // step 2: retrieve bounds of new fragments + return get_all_bounds(ctx, uri); +} + +template +struct ConsolidateOutput { + std::vector fragment_data_; + std::vector>> bounds_; +}; + +template +ConsolidateOutput assert_consolidate_n_wise_bounds( + const Context& ctx, + const std::string& array_uri, + const std::vector& input_fragment_data, + uint64_t fan_in) { + const auto actual_bounds = consolidate_n_wise(ctx, array_uri, fan_in); + + Array forread(ctx, array_uri, TILEDB_READ); + const uint64_t tile_stride = forread.schema().capacity(); + + std::vector output_fragments; + for (uint64_t f = 0; f < input_fragment_data.size(); f += fan_in) { + F output_fragment; + for (uint64_t ff = f; + ff < std::min(f + fan_in, input_fragment_data.size()); + ff++) { + output_fragment.extend(input_fragment_data[ff]); + } + + output_fragments.push_back( + make_global_order(forread, output_fragment, sm::Layout::UNORDERED)); + } + + ASSERTER(output_fragments.size() == actual_bounds.size()); + for (uint64_t f = 0; f < output_fragments.size(); f++) { + const uint64_t num_tiles = actual_bounds[f].size(); + ASSERTER( + num_tiles == + (output_fragments[f].size() + tile_stride - 1) / tile_stride); + + for (size_t t = 0; t < num_tiles; t++) { + const uint64_t lbi = t * tile_stride; + const uint64_t ubi = + std::min( + (t + 1) * tile_stride, output_fragments[f].size()) - + 1; + + const auto lbexpect = tuple_index(output_fragments[f].dimensions(), lbi); + const auto ubexpect = tuple_index(output_fragments[f].dimensions(), ubi); + + const auto [lbactual, ubactual] = actual_bounds[f][t]; + ASSERTER(lbexpect == lbactual); + ASSERTER(ubexpect == ubactual); + } + } + + return ConsolidateOutput{ + .fragment_data_ = output_fragments, .bounds_ = actual_bounds}; +} + +TEST_CASE( + "Fragment metadata global order bounds: 1D fixed consolidation", + "[fragment_info][global-order]") { + VFSTestSetup vfs_test_setup; + const auto array_uri = vfs_test_setup.array_uri( + "fragment_metadata_global_order_bounds_1d_fixed_consolidation"); + + const templates::Dimension dimension( + templates::Domain(0, 1024 * 8), 16); + + Context ctx = vfs_test_setup.ctx(); + + templates::ddl::create_array( + array_uri, + ctx, + std::tuple&>{dimension}, + std::vector>{}, + TILEDB_ROW_MAJOR, + TILEDB_ROW_MAJOR, + 8, + false); + + DeleteArrayGuard delarray(ctx.ptr().get(), array_uri.c_str()); + + using Fragment = templates::Fragment1D; + + SECTION("Non-overlapping") { + std::vector fs; + for (uint64_t f = 0; f < 8; f++) { + Fragment input; + input.resize(8); + std::iota(input.dimension().begin(), input.dimension().end(), 1 + f * 8); + fs.push_back(input); + } + + const auto fragment_bounds = + assert_written_bounds( + vfs_test_setup.ctx(), array_uri, fs); + REQUIRE(fragment_bounds.size() == fs.size()); + + SECTION("Pairs") { + const auto pairwise = assert_consolidate_n_wise_bounds< + tiledb::test::AsserterCatch, + Fragment>(ctx, array_uri, fs, 2); + CHECK(pairwise.bounds_.size() == fs.size() / 2); + + // each new fragment should have two tiles each, + // and since they are ascending they should just be a concatenation + CHECK( + pairwise.bounds_[0] == + std::vector>{ + fragment_bounds[0][0], fragment_bounds[1][0]}); + CHECK( + pairwise.bounds_[1] == + std::vector>{ + fragment_bounds[2][0], fragment_bounds[3][0]}); + CHECK( + pairwise.bounds_[2] == + std::vector>{ + fragment_bounds[4][0], fragment_bounds[5][0]}); + CHECK( + pairwise.bounds_[3] == + std::vector>{ + fragment_bounds[6][0], fragment_bounds[7][0]}); + + // run another round, now each should have four tiles + const auto quadwise = assert_consolidate_n_wise_bounds< + tiledb::test::AsserterCatch, + Fragment>(ctx, array_uri, pairwise.fragment_data_, 2); + CHECK(quadwise.bounds_.size() == 2); + CHECK( + quadwise.bounds_[0] == std::vector>{ + fragment_bounds[0][0], + fragment_bounds[1][0], + fragment_bounds[2][0], + fragment_bounds[3][0]}); + CHECK( + quadwise.bounds_[1] == std::vector>{ + fragment_bounds[4][0], + fragment_bounds[5][0], + fragment_bounds[6][0], + fragment_bounds[7][0]}); + + // run final round + const auto octwise = assert_consolidate_n_wise_bounds< + tiledb::test::AsserterCatch, + Fragment>(ctx, array_uri, quadwise.fragment_data_, 2); + CHECK(octwise.bounds_.size() == 1); + CHECK( + octwise.bounds_[0] == std::vector>{ + fragment_bounds[0][0], + fragment_bounds[1][0], + fragment_bounds[2][0], + fragment_bounds[3][0], + fragment_bounds[4][0], + fragment_bounds[5][0], + fragment_bounds[6][0], + fragment_bounds[7][0]}); + } + + SECTION("Triples") { + const auto triwise = assert_consolidate_n_wise_bounds< + tiledb::test::AsserterCatch, + Fragment>(ctx, array_uri, fs, 3); + CHECK(triwise.bounds_.size() == 3); + + // see notes above + CHECK( + triwise.bounds_[0] == std::vector>{ + fragment_bounds[0][0], + fragment_bounds[1][0], + fragment_bounds[2][0]}); + CHECK( + triwise.bounds_[1] == std::vector>{ + fragment_bounds[3][0], + fragment_bounds[4][0], + fragment_bounds[5][0]}); + CHECK( + triwise.bounds_[2] == + std::vector>{ + fragment_bounds[6][0], fragment_bounds[7][0]}); + + const auto ninewise = assert_consolidate_n_wise_bounds< + tiledb::test::AsserterCatch, + Fragment>(ctx, array_uri, triwise.fragment_data_, 3); + + CHECK(ninewise.bounds_.size() == 1); + CHECK( + ninewise.bounds_[0] == std::vector>{ + fragment_bounds[0][0], + fragment_bounds[1][0], + fragment_bounds[2][0], + fragment_bounds[3][0], + fragment_bounds[4][0], + fragment_bounds[5][0], + fragment_bounds[6][0], + fragment_bounds[7][0]}); + } + } + + auto tile = [](uint64_t lb, uint64_t ub) -> Bounds { + return std::make_pair(std::make_tuple(lb), std::make_tuple(ub)); + }; + + SECTION("Interleaving") { + std::vector fs; + for (uint64_t f = 0; f < 8; f++) { + Fragment input; + input.resize(8); + for (uint64_t c = 0; c < 8; c++) { + input.dimension()[c] = (8 * c + 1 + f); + } + fs.push_back(input); + } + + const auto fragment_bounds = + assert_written_bounds( + vfs_test_setup.ctx(), array_uri, fs); + REQUIRE(fragment_bounds.size() == fs.size()); + + CHECK( + fragment_bounds == std::vector>>{ + {tile(1, 57)}, + {tile(2, 58)}, + {tile(3, 59)}, + {tile(4, 60)}, + {tile(5, 61)}, + {tile(6, 62)}, + {tile(7, 63)}, + {tile(8, 64)}}); + + SECTION("Pairs") { + const auto pairwise = + assert_consolidate_n_wise_bounds( + ctx, array_uri, fs, 2); + + CHECK( + pairwise.bounds_ == std::vector>>{ + {tile(1, 26), tile(33, 58)}, + {tile(3, 28), tile(35, 60)}, + {tile(5, 30), tile(37, 62)}, + {tile(7, 32), tile(39, 64)}}); + + const auto quadwise = + assert_consolidate_n_wise_bounds( + ctx, array_uri, pairwise.fragment_data_, 2); + CHECK( + quadwise.bounds_ == + std::vector>>{ + {tile(1, 12), tile(17, 28), tile(33, 44), tile(49, 60)}, + {tile(5, 16), tile(21, 32), tile(37, 48), tile(53, 64)}, + }); + + const auto octwise = + assert_consolidate_n_wise_bounds( + ctx, array_uri, quadwise.fragment_data_, 2); + CHECK( + octwise.bounds_ == std::vector>>{ + {tile(1, 8), + tile(9, 16), + tile(17, 24), + tile(25, 32), + tile(33, 40), + tile(41, 48), + tile(49, 56), + tile(57, 64)}}); + } + + SECTION("Triples") { + const auto triwise = assert_consolidate_n_wise_bounds< + tiledb::test::AsserterCatch, + Fragment>(ctx, array_uri, fs, 3); + + CHECK( + triwise.bounds_ == ArrayBounds{ + {tile(1, 18), tile(19, 41), tile(42, 59)}, + {tile(4, 21), tile(22, 44), tile(45, 62)}, + {tile(7, 32), tile(39, 64)}}); + + const auto ninewise = assert_consolidate_n_wise_bounds< + tiledb::test::AsserterCatch, + Fragment>(ctx, array_uri, triwise.fragment_data_, 3); + + CHECK( + ninewise.bounds_ == std::vector>>{ + {tile(1, 8), + tile(9, 16), + tile(17, 24), + tile(25, 32), + tile(33, 40), + tile(41, 48), + tile(49, 56), + tile(57, 64)}}); + } + } +} + +template +void rapidcheck_instance_consolidation( + const Context& ctx, + const std::string& array_uri, + uint64_t fan_in, + const std::vector& input) { + Array forread(ctx, array_uri, TILEDB_READ); + std::vector global_order_fragments; + for (const auto& fragment : input) { + global_order_fragments.push_back( + make_global_order(forread, fragment, sm::Layout::UNORDERED)); + } + + ConsolidateOutput state; + state.fragment_data_ = global_order_fragments; + state.bounds_ = assert_written_bounds( + ctx, array_uri, global_order_fragments, sm::Layout::GLOBAL_ORDER); + + while (state.bounds_.size() > 1) { + state = + assert_consolidate_n_wise_bounds( + ctx, array_uri, state.fragment_data_, fan_in); + } +} + +TEST_CASE( + "Fragment metadata global order bounds: 1D var consolidation", + "[fragment_info][global-order]") { + VFSTestSetup vfs_test_setup; + const auto array_uri = vfs_test_setup.array_uri( + "fragment_metadata_global_order_bounds_1d_var_consolidation"); + + const bool allow_dups = GENERATE(true, false); + + const templates::Dimension dimension; + + Context ctx = vfs_test_setup.ctx(); + + templates::ddl::create_array( + array_uri, + ctx, + std::tuple&>{ + dimension}, + std::vector>{}, + TILEDB_ROW_MAJOR, + TILEDB_ROW_MAJOR, + 8, + allow_dups); + + DeleteArrayGuard delarray(ctx.ptr().get(), array_uri.c_str()); + + using Fragment = Fragment1DVar; + + const uint64_t num_cells_per_fragment = 16; + std::vector input; + for (uint64_t f = 0; f < 8; f++) { + Fragment fdata; + for (uint64_t c = 0; c < num_cells_per_fragment; c++) { + const std::string value = std::to_string(c + f * num_cells_per_fragment); + fdata.dimension().push_back( + templates::StringDimensionCoordView(value.data(), value.size())); + } + input.push_back(fdata); + } + + auto tile = [](std::string_view lb, std::string_view ub) -> Bounds { + return std::make_pair( + std::make_tuple( + templates::StringDimensionCoordType(lb.begin(), lb.end())), + std::make_tuple( + templates::StringDimensionCoordType(ub.begin(), ub.end()))); + }; + + const auto fragment_bounds = + assert_written_bounds( + ctx, array_uri, input, sm::Layout::UNORDERED); + + REQUIRE( + fragment_bounds == ArrayBounds{ + {tile("0", "15"), tile("2", "9")}, + {tile("16", "23"), tile("24", "31")}, + {tile("32", "39"), tile("40", "47")}, + {tile("48", "55"), tile("56", "63")}, + {tile("64", "71"), tile("72", "79")}, + {tile("80", "87"), tile("88", "95")}, + {tile("100", "107"), tile("108", "99")}, + {tile("112", "119"), tile("120", "127")}}); + + SECTION("Pairs") { + const auto pairwise = + assert_consolidate_n_wise_bounds( + ctx, array_uri, input, 2); + + CHECK( + pairwise.bounds_ == ArrayBounds{ + {tile("0", "15"), + tile("16", "22"), + tile("23", "3"), + tile("30", "9")}, + {tile("32", "39"), + tile("40", "47"), + tile("48", "55"), + tile("56", "63")}, + {tile("64", "71"), + tile("72", "79"), + tile("80", "87"), + tile("88", "95")}, + {tile("100", "107"), + tile("108", "115"), + tile("116", "123"), + tile("124", "99")}}); + + const auto quadwise = + assert_consolidate_n_wise_bounds( + ctx, array_uri, pairwise.fragment_data_, 2); + + CHECK( + quadwise.bounds_ == ArrayBounds{ + {tile("0", "15"), + tile("16", "22"), + tile("23", "3"), + tile("30", "37"), + tile("38", "44"), + tile("45", "51"), + tile("52", "59"), + tile("6", "9")}, + {tile("100", "107"), + tile("108", "115"), + tile("116", "123"), + tile("124", "67"), + tile("68", "75"), + tile("76", "83"), + tile("84", "91"), + tile("92", "99")}}); + + const auto octwise = + assert_consolidate_n_wise_bounds( + ctx, array_uri, quadwise.fragment_data_, 2); + + CHECK( + octwise.bounds_ == ArrayBounds{ + {tile("0", "104"), + tile("105", "111"), + tile("112", "119"), + tile("12", "126"), + tile("127", "19"), + tile("2", "26"), + tile("27", "33"), + tile("34", "40"), + tile("41", "48"), + tile("49", "55"), + tile("56", "62"), + tile("63", "7"), + tile("70", "77"), + tile("78", "84"), + tile("85", "91"), + tile("92", "99")}}); + } + + SECTION("Triples") { + const auto triwise = + assert_consolidate_n_wise_bounds( + ctx, array_uri, input, 3); + + CHECK( + triwise.bounds_ == ArrayBounds{ + {tile("0", "15"), + tile("16", "22"), + tile("23", "3"), + tile("30", "37"), + tile("38", "44"), + tile("45", "9")}, + {tile("48", "55"), + tile("56", "63"), + tile("64", "71"), + tile("72", "79"), + tile("80", "87"), + tile("88", "95")}, + {tile("100", "107"), + tile("108", "115"), + tile("116", "123"), + tile("124", "99")}}); + + const auto ninewise = + assert_consolidate_n_wise_bounds( + ctx, array_uri, triwise.fragment_data_, 3); + + CHECK( + ninewise.bounds_ == ArrayBounds{ + {tile("0", "104"), + tile("105", "111"), + tile("112", "119"), + tile("12", "126"), + tile("127", "19"), + tile("2", "26"), + tile("27", "33"), + tile("34", "40"), + tile("41", "48"), + tile("49", "55"), + tile("56", "62"), + tile("63", "7"), + tile("70", "77"), + tile("78", "84"), + tile("85", "91"), + tile("92", "99")}}); + } +} + +TEST_CASE( + "Fragment metadata global order bounds: 1D fixed consolidation rapidcheck", + "[fragment_info][global-order][rapidcheck]") { + VFSTestSetup vfs_test_setup; + const auto array_uri = vfs_test_setup.array_uri( + "fragment_metadata_global_order_bounds_1d_fixed_consolidation"); + + const templates::Dimension dimension( + templates::Domain(0, 1024 * 8), 16); + + Context ctx = vfs_test_setup.ctx(); + + auto temp_array = [&](bool allow_dups) { + templates::ddl::create_array( + array_uri, + ctx, + std::tuple&>{dimension}, + std::vector>{}, + TILEDB_ROW_MAJOR, + TILEDB_ROW_MAJOR, + 8, + allow_dups); + + return DeleteArrayGuard(ctx.ptr().get(), array_uri.c_str()); + }; + + rc::prop("1D fixed consolidation", [&](bool allow_dups) { + uint64_t fan_in = *rc::gen::inRange(2, 8); + auto fragments = *rc::gen::suchThat( + rc::gen::container>( + rc::make_fragment_1d(allow_dups, dimension.domain)), + [](auto value) { return value.size() > 1; }); + + auto arrayguard = temp_array(allow_dups); + rapidcheck_instance_consolidation( + ctx, array_uri, fan_in, fragments); + }); +} + +TEST_CASE( + "Fragment metadata global order bounds: 1D var consolidation rapidcheck", + "[fragment_info][global-order][rapidcheck]") { + VFSTestSetup vfs_test_setup; + const auto array_uri = vfs_test_setup.array_uri( + "fragment_metadata_global_order_bounds_1d_fixed_consolidation"); + + const templates::Dimension dimension; + + Context ctx = vfs_test_setup.ctx(); + + auto temp_array = [&](bool allow_dups) { + templates::ddl::create_array( + array_uri, + ctx, + std::tuple&>{ + dimension}, + std::vector>{}, + TILEDB_ROW_MAJOR, + TILEDB_ROW_MAJOR, + 8, + allow_dups); + + return DeleteArrayGuard(ctx.ptr().get(), array_uri.c_str()); + }; + + using F = Fragment1DVar; + + rc::prop("1D var consolidation", [&](bool allow_dups) { + uint64_t fan_in = *rc::gen::inRange(2, 8); + auto fragments = *rc::gen::suchThat( + rc::gen::container>( + rc::make_fragment_1d( + allow_dups)), + [](auto value) { return value.size() > 1; }); + + auto arrayguard = temp_array(allow_dups); + rapidcheck_instance_consolidation(ctx, array_uri, fan_in, fragments); + }); +} + +/** + * Rapidcheck bounds consolidation test using the VCF 2025 data model + * (3D sparse array with chromosome/position/sample dimensions) + */ +TEST_CASE( + "Fragment metadata global order bounds: 3D vcf consolidation rapidcheck", + "[fragment_info][global-order][rapidcheck]") { + VFSTestSetup vfs_test_setup; + const auto array_uri = vfs_test_setup.array_uri( + "fragment_metadata_global_order_bounds_3d_vcf_consolidation"); + + const templates::Domain domain_sample(0, 10000); + + const templates::Dimension d_chromosome; + const templates::Dimension d_position(domain_sample, 32); + const templates::Dimension d_sample; + + Context ctx = vfs_test_setup.ctx(); + + auto temp_array = [&](bool allow_dups) { + templates::ddl::create_array< + Datatype::STRING_ASCII, + Datatype::UINT32, + Datatype::STRING_ASCII>( + array_uri, + ctx, + std::tie(d_chromosome, d_position, d_sample), + std::vector>{}, + TILEDB_ROW_MAJOR, + TILEDB_ROW_MAJOR, + 8, + allow_dups); + + return DeleteArrayGuard(ctx.ptr().get(), array_uri.c_str()); + }; + + using F = FragmentVcf2025; + + rc::prop("3D vcf2025 consolidation", [&](bool allow_dups) { + uint64_t fan_in = *rc::gen::inRange(2, 8); + auto fragments = *rc::gen::suchThat( + rc::gen::container>(rc::make_fragment_3d< + templates::StringDimensionCoordType, + uint32_t, + templates::StringDimensionCoordType>( + allow_dups, std::nullopt, domain_sample, std::nullopt)), + [](auto value) { return value.size() > 1; }); + + auto arrayguard = temp_array(allow_dups); + rapidcheck_instance_consolidation(ctx, array_uri, fan_in, fragments); + }); +} diff --git a/test/support/rapidcheck/array_templates.h b/test/support/rapidcheck/array_templates.h index 37762a9ba6b..b4550027203 100644 --- a/test/support/rapidcheck/array_templates.h +++ b/test/support/rapidcheck/array_templates.h @@ -46,9 +46,7 @@ using namespace tiledb::test::templates; template Gen> make_fragment_1d( - bool allow_duplicates, const Domain& d) { - auto coord = make_coordinate(d); - + bool allow_duplicates, Gen coord) { auto cell = gen::tuple(coord, gen::arbitrary()...); using Cell = std::tuple; @@ -83,6 +81,19 @@ Gen> make_fragment_1d( }); } +template +Gen> make_fragment_1d( + bool allow_duplicates) { + return make_fragment_1d(allow_duplicates, gen::arbitrary()); +} + +template +Gen> make_fragment_1d( + bool allow_duplicates, const Domain& d) { + auto coord = make_coordinate(d); + return make_fragment_1d(allow_duplicates, coord); +} + template Gen> make_fragment_2d( bool allow_duplicates, diff --git a/test/support/rapidcheck/show/query_ast.cc b/test/support/rapidcheck/show/query_ast.cc index f895667de4c..efeae6dca51 100644 --- a/test/support/rapidcheck/show/query_ast.cc +++ b/test/support/rapidcheck/show/query_ast.cc @@ -42,6 +42,26 @@ namespace rc::detail { +template +void showDomain(const templates::Domain& domain, std::ostream& os) { + os << "[" << domain.lower_bound << ", " << domain.upper_bound << "]"; +} + +void showValue(const templates::Domain& domain, std::ostream& os) { + showDomain(domain, os); +} +void showValue(const templates::Domain& domain, std::ostream& os) { + showDomain(domain, os); +} +void showValue(const templates::Domain& domain, std::ostream& os) { + showDomain(domain, os); +} + +template +void showQueryBuffers(const templates::query_buffers& qb, std::ostream& os) { + show(qb.values_, os); +} + void showValue(const tiledb::sm::ASTNode& node, std::ostream& os) { const tiledb::sm::ASTNodeVal* valnode = static_cast(&node); @@ -80,3 +100,11 @@ void showValue(const tiledb::sm::ASTNode& node, std::ostream& os) { } } // namespace rc::detail + +namespace rc { + +void showValue(const templates::query_buffers& qb, std::ostream& os) { + detail::showQueryBuffers(qb, os); +} + +} // namespace rc diff --git a/test/support/src/array_schema_templates.h b/test/support/src/array_schema_templates.h index bd2b77059a8..f1a5503e9ea 100644 --- a/test/support/src/array_schema_templates.h +++ b/test/support/src/array_schema_templates.h @@ -158,6 +158,17 @@ struct Dimension { } }; +template <> +struct Dimension { + using value_type = StringDimensionCoordType; + + static constexpr tiledb::sm::Datatype DATATYPE = + tiledb::sm::Datatype::STRING_ASCII; + + Dimension() { + } +}; + template struct static_attribute {}; diff --git a/test/support/src/array_templates.h b/test/support/src/array_templates.h index 9dc422bc48a..9f923a0f95b 100644 --- a/test/support/src/array_templates.h +++ b/test/support/src/array_templates.h @@ -328,7 +328,7 @@ struct query_buffers { std::vector values_; - query_buffers() { + constexpr query_buffers() { } query_buffers(const self_type& other) @@ -499,7 +499,7 @@ struct query_buffers> { std::vector values_; std::vector validity_; - query_buffers() { + constexpr query_buffers() { } query_buffers(const self_type& other) = default; @@ -742,7 +742,7 @@ struct query_buffers> { std::vector values_; std::vector offsets_; - query_buffers() { + constexpr query_buffers() { } query_buffers(const self_type& other) = default; @@ -948,7 +948,7 @@ struct query_buffers>> { std::vector offsets_; std::vector validity_; - query_buffers() { + constexpr query_buffers() { } query_buffers(const self_type& other) = default; @@ -1172,15 +1172,18 @@ struct Fragment { using AttributeBuffersConstRef = const_ref_tuple_query_buffers; + static constexpr size_t NUM_DIMENSIONS = + std::tuple_size::value; + static constexpr size_t NUM_ATTRIBUTES = + std::tuple_size::value; + DimensionBuffers dims_; AttributeBuffers atts_; uint64_t num_cells() const { - static_assert( - std::tuple_size::value > 0 || - std::tuple_size::value > 0); + static_assert(NUM_DIMENSIONS > 0 || NUM_ATTRIBUTES > 0); - if constexpr (std::tuple_size::value == 0) { + if constexpr (NUM_DIMENSIONS == 0) { return std::get<0>(atts_).num_cells(); } else { return std::get<0>(dims_).num_cells(); diff --git a/tiledb/api/c_api/fragment_info/fragment_info_api.cc b/tiledb/api/c_api/fragment_info/fragment_info_api.cc index 4a66a5e869c..3db174209e2 100644 --- a/tiledb/api/c_api/fragment_info/fragment_info_api.cc +++ b/tiledb/api/c_api/fragment_info/fragment_info_api.cc @@ -316,6 +316,36 @@ capi_return_t tiledb_fragment_info_get_mbr_var_from_name( return TILEDB_OK; } +capi_return_t tiledb_fragment_info_get_global_order_lower_bound( + tiledb_fragment_info_t* fragment_info, + uint32_t fragment_id, + uint32_t mbr_id, + size_t* dimension_sizes, + void** dimensions) { + ensure_fragment_info_is_valid(fragment_info); + ensure_output_pointer_is_valid(dimensions); + + throw_if_not_ok(fragment_info->get_global_order_lower_bound( + fragment_id, mbr_id, dimension_sizes, dimensions)); + + return TILEDB_OK; +} + +capi_return_t tiledb_fragment_info_get_global_order_upper_bound( + tiledb_fragment_info_t* fragment_info, + uint32_t fragment_id, + uint32_t mbr_id, + size_t* dimension_sizes, + void** dimensions) { + ensure_fragment_info_is_valid(fragment_info); + ensure_output_pointer_is_valid(dimensions); + + throw_if_not_ok(fragment_info->get_global_order_upper_bound( + fragment_id, mbr_id, dimension_sizes, dimensions)); + + return TILEDB_OK; +} + capi_return_t tiledb_fragment_info_get_cell_num( tiledb_fragment_info_t* fragment_info, uint32_t fid, uint64_t* cell_num) { ensure_fragment_info_is_valid(fragment_info); @@ -709,6 +739,32 @@ CAPI_INTERFACE( ctx, fragment_info, fid, mid, dim_name, start, end); } +CAPI_INTERFACE( + fragment_info_get_global_order_lower_bound, + tiledb_ctx_t* ctx, + tiledb_fragment_info_t* fragment_info, + uint32_t fragment_id, + uint32_t mbr_id, + size_t* dimension_sizes, + void** dimensions) { + return api_entry_context< + tiledb::api::tiledb_fragment_info_get_global_order_lower_bound>( + ctx, fragment_info, fragment_id, mbr_id, dimension_sizes, dimensions); +} + +CAPI_INTERFACE( + fragment_info_get_global_order_upper_bound, + tiledb_ctx_t* ctx, + tiledb_fragment_info_t* fragment_info, + uint32_t fragment_id, + uint32_t mbr_id, + size_t* dimension_sizes, + void** dimensions) { + return api_entry_context< + tiledb::api::tiledb_fragment_info_get_global_order_upper_bound>( + ctx, fragment_info, fragment_id, mbr_id, dimension_sizes, dimensions); +} + CAPI_INTERFACE( fragment_info_get_cell_num, tiledb_ctx_t* ctx, diff --git a/tiledb/api/c_api/fragment_info/fragment_info_api_external.h b/tiledb/api/c_api/fragment_info/fragment_info_api_external.h index d08774a5712..438476388e4 100644 --- a/tiledb/api/c_api/fragment_info/fragment_info_api_external.h +++ b/tiledb/api/c_api/fragment_info/fragment_info_api_external.h @@ -713,6 +713,87 @@ TILEDB_EXPORT capi_return_t tiledb_fragment_info_get_mbr_var_from_name( void* start, void* end) TILEDB_NOEXCEPT; +/** + * Retrieves the minimum coordinate in global order for a + * particular bounding rectangle in a fragment. + * + * **Example:** + * + * @code{.c} + * int32_t cell_id; + * size_t gene_name_size; + * char gene_name[MAX_GENE_NAME_LEN]; + * + * size_t dimension_sizes[] = { nullptr, &gene_name_size }; + * void *dimensions[] = { &cell_id, &gene_name[0] }; + * + * tiledb_fragment_info_get_global_order_lower_bound(ctx, + * fragment_info, 0, 0, + * &dimension_sizes[0], + * &dimensions[0]); + * @endcode + * + * If any dimension is `NULL` then only the size is returned. This enables + * asking for the size of the bounds of variable-length dimensions. + * + * @param[in] ctx The TileDB context + * @param[in] fragment_info The fragment info object. + * @param[in] fragment_id The index of the fragment of interest + * @param[in] mbr_id The mbr of the fragment of interest + * @param[in/out] dimension_sizes An array of pointers to sizes to store the + * length of the value of each variable-length dimension + * @param[in/out] dimensions An array of pointers to buffers to store the value + * from each dimension + * @return TILEDB_OK if successful, TILEDB_ERR if an error occurs or if the + * fragment version is prior + */ +TILEDB_EXPORT capi_return_t tiledb_fragment_info_get_global_order_lower_bound( + tiledb_ctx_t* ctx, + tiledb_fragment_info_t* fragment_info, + uint32_t fragment_id, + uint32_t mbr_id, + size_t* dimension_sizes, + void** dimensions) TILEDB_NOEXCEPT; + +/** + * Retrieves the maximum coordinate in global order for a + * particular bounding rectangle in a fragment. + * + * **Example:** + * + * @code{.c} + * int32_t cell_id; + * size_t gene_name_size; + * char gene_name[MAX_GENE_NAME_LEN]; + * + * size_t dimension_sizes[] = { nullptr, &gene_name_size }; + * void *dimensions[] = { &cell_id, &gene_name[0] }; + * + * tiledb_fragment_info_get_global_order_upper_bound(ctx, + * fragment_info, 0, 0, + * &dimension_sizes[0], + * &dimensions[0]); + * @endcode + * + * @param[in] ctx The TileDB context + * @param[in] fragment_info The fragment info object. + * @param[in] fragment_id The index of the fragment of interest + * @param[in] mbr_id The mbr of the fragment of interest + * @param[in/out] dimension_sizes An array of pointers to sizes to store the + * length of the value of each variable-length dimension + * @param[in/out] dimensions An array of pointers to buffers to store the value + * from each dimension + * @return TILEDB_OK if successful, TILEDB_ERR if an error occurs or if the + * fragment version is prior + */ +TILEDB_EXPORT capi_return_t tiledb_fragment_info_get_global_order_upper_bound( + tiledb_ctx_t* ctx, + tiledb_fragment_info_t* fragment_info, + uint32_t fragment_id, + uint32_t mbr_id, + size_t* dimension_sizes, + void** dimensions) TILEDB_NOEXCEPT; + /** * Retrieves the number of cells written to the fragment by the user. * diff --git a/tiledb/api/c_api/fragment_info/fragment_info_api_internal.h b/tiledb/api/c_api/fragment_info/fragment_info_api_internal.h index 1140cb33a72..c4e02d5467c 100644 --- a/tiledb/api/c_api/fragment_info/fragment_info_api_internal.h +++ b/tiledb/api/c_api/fragment_info/fragment_info_api_internal.h @@ -151,6 +151,24 @@ struct tiledb_fragment_info_handle_t fid, mid, dim_name, start_size, end_size); } + Status get_global_order_lower_bound( + uint32_t fid, + uint32_t mid, + size_t* dimension_sizes, + void** dimension_ptrs) { + return fragment_info_->get_global_order_lower_bound( + fid, mid, dimension_sizes, dimension_ptrs); + } + + Status get_global_order_upper_bound( + uint32_t fid, + uint32_t mid, + size_t* dimension_sizes, + void** dimension_ptrs) { + return fragment_info_->get_global_order_upper_bound( + fid, mid, dimension_sizes, dimension_ptrs); + } + Status get_non_empty_domain(uint32_t fid, uint32_t did, void* domain) const { return fragment_info_->get_non_empty_domain(fid, did, domain); } diff --git a/tiledb/common/memory_tracker.cc b/tiledb/common/memory_tracker.cc index 27793715ff2..3cbafe5099c 100644 --- a/tiledb/common/memory_tracker.cc +++ b/tiledb/common/memory_tracker.cc @@ -97,6 +97,10 @@ std::string memory_type_to_str(MemoryType type) { return "TileMaxVals"; case MemoryType::TILE_MIN_VALS: return "TileMinVals"; + case MemoryType::TILE_GLOBAL_ORDER_MAX_VALS: + return "TileGlobalOrderMaxVals"; + case MemoryType::TILE_GLOBAL_ORDER_MIN_VALS: + return "TileGlobalOrderMinVals"; case MemoryType::TILE_NULL_COUNTS: return "TileNullCounts"; case MemoryType::TILE_OFFSETS: diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index bc109728a7e..43a10256b7b 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -133,6 +133,8 @@ enum class MemoryType { TILE_HILBERT_VALUES, TILE_MAX_VALS, TILE_MIN_VALS, + TILE_GLOBAL_ORDER_MIN_VALS, + TILE_GLOBAL_ORDER_MAX_VALS, TILE_NULL_COUNTS, TILE_OFFSETS, TILE_SUMS, diff --git a/tiledb/sm/array_schema/dimension.h b/tiledb/sm/array_schema/dimension.h index 3f9f62be1f8..54e23a70f10 100644 --- a/tiledb/sm/array_schema/dimension.h +++ b/tiledb/sm/array_schema/dimension.h @@ -743,6 +743,17 @@ class Dimension { return cell_val_num_ == constants::var_num; } + /** + * @return the size in bytes of one cell for this dimension, or + * `constants::var_size` if variable-length + */ + uint64_t cell_size() const { + if (var_size()) { + return constants::var_size; + } + return cell_val_num_ * datatype_size(type_); + } + class DimensionDispatch { public: DimensionDispatch(const Dimension& base) diff --git a/tiledb/sm/array_schema/domain.cc b/tiledb/sm/array_schema/domain.cc index bde13d0ed98..324fd999687 100644 --- a/tiledb/sm/array_schema/domain.cc +++ b/tiledb/sm/array_schema/domain.cc @@ -273,10 +273,9 @@ int Domain::cell_order_cmp( const type::DomainDataRef& left, const type::DomainDataRef& right) const { if (cell_order_ == Layout::ROW_MAJOR || cell_order_ == Layout::HILBERT) { for (unsigned d = 0; d < dim_num_; ++d) { - auto res = cell_order_cmp_func_[d]( - dimension_ptr(d), - left.dimension_datum_view(d), - right.dimension_datum_view(d)); + const auto ldatum = left.dimension_datum_view(d); + const auto rdatum = right.dimension_datum_view(d); + auto res = cell_order_cmp(d, ldatum, rdatum); if (res == 1 || res == -1) return res; @@ -284,10 +283,9 @@ int Domain::cell_order_cmp( } } else { // COL_MAJOR for (unsigned d = dim_num_ - 1;; --d) { - auto res = cell_order_cmp_func_[d]( - dimension_ptr(d), - left.dimension_datum_view(d), - right.dimension_datum_view(d)); + const auto ldatum = left.dimension_datum_view(d); + const auto rdatum = right.dimension_datum_view(d); + auto res = cell_order_cmp(d, ldatum, rdatum); if (res == 1 || res == -1) return res; @@ -491,15 +489,24 @@ bool Domain::has_dimension(const std::string& name) const { return false; } -unsigned Domain::get_dimension_index(const std::string& name) const { +std::optional Domain::dimension_index(const std::string& name) const { for (unsigned d = 0; d < dim_num_; ++d) { if (dimension_ptrs_[d]->name() == name) { return d; } } - throw std::invalid_argument( - "Cannot get dimension index; Invalid dimension name"); + return std::nullopt; +} + +unsigned Domain::get_dimension_index(const std::string& name) const { + const auto maybe = dimension_index(name); + if (maybe.has_value()) { + return maybe.value(); + } else { + throw std::invalid_argument( + "Cannot get dimension index; Invalid dimension name"); + } } bool Domain::null_tile_extents() const { diff --git a/tiledb/sm/array_schema/domain.h b/tiledb/sm/array_schema/domain.h index 4132f2dbc6e..0e8f6911f6c 100644 --- a/tiledb/sm/array_schema/domain.h +++ b/tiledb/sm/array_schema/domain.h @@ -394,9 +394,16 @@ class Domain { * * @param name Name of dimension to check for * @return Dimension index + * @throws if the name is not a dimension */ unsigned get_dimension_index(const std::string& name) const; + /** + * @return the index in the domain of the requested dimension, or + * `std::nullopt` if not found + */ + std::optional dimension_index(const std::string& name) const; + /** Returns true if at least one dimension has null tile extent. */ bool null_tile_extents() const; diff --git a/tiledb/sm/cpp_api/fragment_info.h b/tiledb/sm/cpp_api/fragment_info.h index a398c68ffaa..61fcfbf1c00 100644 --- a/tiledb/sm/cpp_api/fragment_info.h +++ b/tiledb/sm/cpp_api/fragment_info.h @@ -279,6 +279,98 @@ class FragmentInfo { return std::make_pair(start, end); } + /** + * Returns the minimum coordinate in global order for a particular bounding + * rectangle in a fragment. The returned value contains one element per + * dimension. + * + * @throws if the fragment index `fid` is invalid; if the bounding rectangle + * index `mid` is invalid; if the fragment is not a sparse fragment; or if the + * fragment was written in a format version which does not contain the + * bounding rectangle global order bounds. + */ + std::vector> global_order_lower_bound( + uint32_t fid, uint32_t mid) const { + std::vector dimension_sizes; + std::vector dimension_ptrs; + dimension_sizes.resize(array_schema(fid).domain().ndim()); + dimension_ptrs.resize(dimension_sizes.size()); + + // 1. get sizes + ctx_.get().handle_error(tiledb_fragment_info_get_global_order_lower_bound( + ctx_.get().ptr().get(), + fragment_info_.get(), + fid, + mid, + dimension_sizes.data(), + dimension_ptrs.data())); + + // 2. get data + std::vector> dimension_bufs; + dimension_bufs.resize(dimension_sizes.size()); + + for (uint64_t d = 0; d < dimension_sizes.size(); d++) { + dimension_bufs[d].resize(dimension_sizes[d]); + dimension_ptrs[d] = dimension_bufs[d].data(); + } + + ctx_.get().handle_error(tiledb_fragment_info_get_global_order_lower_bound( + ctx_.get().ptr().get(), + fragment_info_.get(), + fid, + mid, + dimension_sizes.data(), + dimension_ptrs.data())); + + return dimension_bufs; + } + + /** + * Returns the maximum coordinate in global order for a particular bounding + * rectangle in a fragment. The returned value contains one element per + * dimension. + * + * @throws if the fragment index `fid` is invalid; if the bounding rectangle + * index `mid` is invalid; if the fragment is not a sparse fragment; or if the + * fragment was written in a format version which does not contain the + * bounding rectangle global order bounds. + */ + std::vector> global_order_upper_bound( + uint32_t fid, uint32_t mid) const { + std::vector dimension_sizes; + std::vector dimension_ptrs; + dimension_sizes.resize(array_schema(fid).domain().ndim()); + dimension_ptrs.resize(dimension_sizes.size()); + + // 1. get sizes + ctx_.get().handle_error(tiledb_fragment_info_get_global_order_upper_bound( + ctx_.get().ptr().get(), + fragment_info_.get(), + fid, + mid, + dimension_sizes.data(), + dimension_ptrs.data())); + + // 2. get data + std::vector> dimension_bufs; + dimension_bufs.resize(dimension_sizes.size()); + + for (uint64_t d = 0; d < dimension_sizes.size(); d++) { + dimension_bufs[d].resize(dimension_sizes[d]); + dimension_ptrs[d] = dimension_bufs[d].data(); + } + + ctx_.get().handle_error(tiledb_fragment_info_get_global_order_upper_bound( + ctx_.get().ptr().get(), + fragment_info_.get(), + fid, + mid, + dimension_sizes.data(), + dimension_ptrs.data())); + + return dimension_bufs; + } + /** Returns the number of fragments. */ uint32_t fragment_num() const { auto& ctx = ctx_.get(); diff --git a/tiledb/sm/fragment/fragment_info.cc b/tiledb/sm/fragment/fragment_info.cc index 87c3c7a76e2..733c6115ad5 100644 --- a/tiledb/sm/fragment/fragment_info.cc +++ b/tiledb/sm/fragment/fragment_info.cc @@ -77,6 +77,7 @@ void FragmentInfo::set_config(const Config& config) { throw FragmentInfoException("[set_config] Cannot set config after load"); } config_.inherit(config); + throw_if_not_ok(set_enc_key_from_config()); } void FragmentInfo::expand_anterior_ndrange( @@ -722,6 +723,127 @@ Status FragmentInfo::get_mbr_var( return get_mbr_var(fid, mid, did, start, end); } +static Status read_global_order_bound_to_user_buffers( + const ArraySchema& schema, + const auto& fixedPart, + const auto& varPart, + uint32_t which_tile, + size_t* dimension_sizes, + void** dimensions) { + const auto ds = schema.domain().dimensions(); + for (uint64_t d = 0; d < ds.size(); d++) { + if (ds[d]->var_size()) { + if (dimension_sizes == nullptr) { + throw FragmentInfoException( + "Cannot get MBR global order bound: Variable-length dimension " + "requires non-NULL dimension_sizes argument"); + } + + std::span offsets( + reinterpret_cast(fixedPart[d].data()), + fixedPart[d].size() / sizeof(uint64_t)); + if (which_tile >= offsets.size()) { + throw FragmentInfoException( + "Cannot get MBR global order bound: Invalid mbr index"); + } else if (which_tile + 1 == offsets.size()) { + dimension_sizes[d] = varPart[d].size() - offsets[which_tile]; + } else { + dimension_sizes[d] = offsets[which_tile + 1] - offsets[which_tile]; + } + + if (dimensions[d]) { + const void* coord = &varPart[d][offsets[which_tile]]; + memcpy(dimensions[d], coord, dimension_sizes[d]); + } + } else { + const uint64_t dimFixedSize = ds[d]->cell_size(); + if (dimFixedSize * which_tile >= fixedPart[d].size()) { + throw FragmentInfoException( + "Cannot get MBR global order bound: Invalid mbr index"); + } + + if (dimension_sizes) { + dimension_sizes[d] = dimFixedSize; + } + + if (dimensions[d]) { + const void* coord = &fixedPart[d].data()[which_tile * dimFixedSize]; + memcpy(dimensions[d], coord, dimFixedSize); + } + } + } + + return Status::Ok(); +} + +Status FragmentInfo::get_global_order_lower_bound( + uint32_t fid, uint32_t mid, size_t* dimension_sizes, void** dimensions) { + ensure_loaded(); + if (fid >= fragment_num()) { + throw FragmentInfoException( + "Cannot get MBR global order bound: Invalid fragment index"); + } + + if (!single_fragment_info_vec_[fid].sparse()) { + throw FragmentInfoException("Cannot get MBR; Fragment is not sparse"); + } + + auto& meta = single_fragment_info_vec_[fid].meta(); + meta->loaded_metadata()->load_fragment_tile_global_order_bounds(enc_key_); + + const auto& fixedPart = + meta->loaded_metadata()->tile_global_order_min_buffer(); + if (fixedPart.empty()) { + throw FragmentInfoException( + "Cannot get MBR global order bound: Unavailable"); + } + + const auto& varPart = + meta->loaded_metadata()->tile_global_order_min_var_buffer(); + + return read_global_order_bound_to_user_buffers( + *meta->array_schema(), + fixedPart, + varPart, + mid, + dimension_sizes, + dimensions); +} + +Status FragmentInfo::get_global_order_upper_bound( + uint32_t fid, uint32_t mid, size_t* dimension_sizes, void** dimensions) { + ensure_loaded(); + if (fid >= fragment_num()) { + throw FragmentInfoException( + "Cannot get MBR global order bound: Invalid fragment index"); + } + + if (!single_fragment_info_vec_[fid].sparse()) { + throw FragmentInfoException("Cannot get MBR; Fragment is not sparse"); + } + + auto& meta = single_fragment_info_vec_[fid].meta(); + meta->loaded_metadata()->load_fragment_tile_global_order_bounds(enc_key_); + + const auto& fixedPart = + meta->loaded_metadata()->tile_global_order_max_buffer(); + if (fixedPart.empty()) { + throw FragmentInfoException( + "Cannot get MBR global order bound: Unavailable"); + } + + const auto& varPart = + meta->loaded_metadata()->tile_global_order_max_var_buffer(); + + return read_global_order_bound_to_user_buffers( + *meta->array_schema(), + fixedPart, + varPart, + mid, + dimension_sizes, + dimensions); +} + Status FragmentInfo::get_version(uint32_t fid, uint32_t* version) const { ensure_loaded(); if (version == nullptr) { @@ -890,6 +1012,8 @@ Status FragmentInfo::load(const ArrayDirectory& array_dir) { if (preload_rtrees & !meta->dense()) { meta->loaded_metadata()->load_rtree(enc_key_); + meta->loaded_metadata()->load_fragment_tile_global_order_bounds( + enc_key_); } return Status::Ok(); diff --git a/tiledb/sm/fragment/fragment_info.h b/tiledb/sm/fragment/fragment_info.h index 2dee1a06e48..dd23baa4124 100644 --- a/tiledb/sm/fragment/fragment_info.h +++ b/tiledb/sm/fragment/fragment_info.h @@ -215,6 +215,28 @@ class FragmentInfo { Status get_mbr_var( uint32_t fid, uint32_t mid, const char* dim_name, void* start, void* end); + /** + * Retrieves a global order lower bound of the fragment with the given index + * for the tile indexed by `mid`. + * + * For fragments of version 23 and newer this is precise. + * Otherwise the minimum bounding rectangle is used as an imprecise lower + * bound. + */ + Status get_global_order_lower_bound( + uint32_t fid, uint32_t mid, size_t* dimension_sizes, void** dimensions); + + /** + * Retrieves a global order upper bound of the fragment with the given index + * for the tile indexed by `mid`. + * + * For fragments of version 23 and newer this is precise. + * Otherwise the minimum bounding rectangle is used as an imprecise upper + * bound. + */ + Status get_global_order_upper_bound( + uint32_t fid, uint32_t mid, size_t* dimension_sizes, void** dimensions); + /** Retrieves the version of the fragment with the given index. */ Status get_version(uint32_t fid, uint32_t* version) const; diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index 6d2b1789caf..47847c5f736 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -57,6 +57,7 @@ #include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/sm/tile/tile.h" #include "tiledb/sm/tile/tile_metadata_generator.h" +#include "tiledb/sm/tile/writer_tile_tuple.h" #include "tiledb/storage_format/serialization/serializers.h" #include "tiledb/type/range/range.h" @@ -296,6 +297,28 @@ void FragmentMetadata::set_tile_max_var( } } +/** + * Converts a buffer of `uint64_t` variable-length data sizes into a buffer + * of `uint64_t` variable-length data offsets, in place. + * + * @return the total size of variable-length data + */ +static uint64_t convert_sizes_to_offsets( + uint64_t cell_start, uint64_t acc, std::span raw_offsets) { + const uint64_t num_offsets = raw_offsets.size() / sizeof(uint64_t); + uint64_t* offsets = + &reinterpret_cast(raw_offsets.data())[cell_start]; + + for (uint64_t i = cell_start; i < num_offsets; i++) { + auto size = *offsets; + *offsets = acc; + ++offsets; + acc += size; + } + + return acc; +} + void FragmentMetadata::convert_tile_min_max_var_sizes_to_offsets( const std::string& name) { auto it = idx_map_.find(name); @@ -303,39 +326,142 @@ void FragmentMetadata::convert_tile_min_max_var_sizes_to_offsets( auto idx = it->second; // Fix the min offsets. - uint64_t offset = loaded_metadata_ptr_->tile_min_var_buffer()[idx].size(); - auto offsets = - (uint64_t*)loaded_metadata_ptr_->tile_min_buffer()[idx].data() + - tile_index_base_; - for (uint64_t i = tile_index_base_; - i < - loaded_metadata_ptr_->tile_min_buffer()[idx].size() / sizeof(uint64_t); - i++) { - auto size = *offsets; - *offsets = offset; - offsets++; - offset += size; - } + const uint64_t min_var_total = convert_sizes_to_offsets( + tile_index_base_, + loaded_metadata_ptr_->tile_min_var_buffer()[idx].size(), + loaded_metadata_ptr_->tile_min_buffer()[idx]); // Allocate min var data buffer. - loaded_metadata_ptr_->tile_min_var_buffer()[idx].resize(offset); + loaded_metadata_ptr_->tile_min_var_buffer()[idx].resize(min_var_total); // Fix the max offsets. - offset = loaded_metadata_ptr_->tile_max_var_buffer()[idx].size(); - offsets = (uint64_t*)loaded_metadata_ptr_->tile_max_buffer()[idx].data() + - tile_index_base_; - for (uint64_t i = tile_index_base_; - i < - loaded_metadata_ptr_->tile_max_buffer()[idx].size() / sizeof(uint64_t); - i++) { - auto size = *offsets; - *offsets = offset; - offsets++; - offset += size; + const uint64_t max_var_total = convert_sizes_to_offsets( + tile_index_base_, + loaded_metadata_ptr_->tile_max_var_buffer()[idx].size(), + loaded_metadata_ptr_->tile_max_buffer()[idx]); + + // Allocate max var data buffer. + loaded_metadata_ptr_->tile_max_var_buffer()[idx].resize(max_var_total); +} + +void FragmentMetadata::set_tile_global_order_bounds_fixed( + const std::string& dim_name, + uint64_t which_tile, + const WriterTileTuple& data) { + iassert(data.cell_num() > 0); + + const auto dim = array_schema_->domain().get_dimension_index(dim_name); + + const auto& tile_min = data.global_order_min(); + iassert(tile_min.has_value()); + + const auto& tile_max = data.global_order_max(); + iassert(tile_max.has_value()); + + const uint64_t tile = which_tile + tile_index_base_; + + if (array_schema_->domain().dimensions()[dim]->var_size()) { + // NB: for now we set a length, and it will be updated to an offset + // via `convert_tile_global_order_bounds_sizes_to_offsets`, + // and then the var data will be written after that + + uint64_t* min_sizes = reinterpret_cast( + loaded_metadata_ptr_->tile_global_order_min_buffer()[dim].data()); + min_sizes[tile] = tile_min.value().size(); + + uint64_t* max_sizes = reinterpret_cast( + loaded_metadata_ptr_->tile_global_order_max_buffer()[dim].data()); + max_sizes[tile] = tile_max.value().size(); + } else { + const uint64_t fixed_size = + array_schema_->domain().dimensions()[dim]->cell_size(); + iassert(tile_min.value().size() == fixed_size); + iassert(tile_max.value().size() == fixed_size); + + const uint64_t offset = fixed_size * tile; + + uint8_t* min_data = + loaded_metadata_ptr_->tile_global_order_min_buffer()[dim].data(); + memcpy(&min_data[offset], tile_min.value().data(), fixed_size); + + uint8_t* max_data = + loaded_metadata_ptr_->tile_global_order_max_buffer()[dim].data(); + memcpy(&max_data[offset], tile_max.value().data(), fixed_size); + } +} + +/** + * Writes the variable-length part of the global order bounds into the fragment + * metadata. + * + * The seqeunce of calls is + * 1) set_tile_global_order_bounds_fixed + * 2) convert_tile_global_order_bounds_sizes_to_offsets + * 3) this + */ +void FragmentMetadata::set_tile_global_order_bounds_var( + const std::string& dim_name, uint64_t wtile, const WriterTileTuple& data) { + const auto dim = array_schema_->domain().get_dimension_index(dim_name); + if (!array_schema_->domain().dimensions()[dim]->var_size()) { + return; + } + + const uint64_t tile = tile_index_base_ + wtile; + + const auto& tile_min = data.global_order_min(); + iassert(tile_min.has_value()); + + const auto& tile_max = data.global_order_max(); + iassert(tile_max.has_value()); + + const uint64_t* min_offsets = reinterpret_cast( + loaded_metadata_ptr_->tile_global_order_min_buffer()[dim].data()); + const uint64_t* max_offsets = reinterpret_cast( + loaded_metadata_ptr_->tile_global_order_max_buffer()[dim].data()); + + const uint64_t min_var_start = min_offsets[tile]; + const uint64_t max_var_start = max_offsets[tile]; + + if (tile_min.value().size()) { + memcpy( + &loaded_metadata_ptr_ + ->tile_global_order_min_var_buffer()[dim][min_var_start], + tile_min.value().data(), + tile_min.value().size()); + } + if (tile_max.value().size()) { + memcpy( + &loaded_metadata_ptr_ + ->tile_global_order_max_var_buffer()[dim][max_var_start], + tile_max.value().data(), + tile_max.value().size()); } +} + +void FragmentMetadata::convert_tile_global_order_bounds_sizes_to_offsets( + const std::string& dim_name) { + // NB: this is dimensions only, so do not use `idx_map_` + const auto idx = array_schema_->domain().get_dimension_index(dim_name); + + // Fix the min offsets. + const uint64_t min_var_total = convert_sizes_to_offsets( + tile_index_base_, + loaded_metadata_ptr_->tile_global_order_min_var_buffer()[idx].size(), + loaded_metadata_ptr_->tile_global_order_min_buffer()[idx]); // Allocate min var data buffer. - loaded_metadata_ptr_->tile_max_var_buffer()[idx].resize(offset); + loaded_metadata_ptr_->tile_global_order_min_var_buffer()[idx].resize( + min_var_total); + + // Fix the max offsets. + const uint64_t max_var_total = convert_sizes_to_offsets( + tile_index_base_, + loaded_metadata_ptr_->tile_global_order_max_var_buffer()[idx].size(), + loaded_metadata_ptr_->tile_global_order_max_buffer()[idx]); + + // Allocate max var data buffer. + loaded_metadata_ptr_->tile_global_order_max_var_buffer()[idx].resize( + max_var_total); } void FragmentMetadata::set_tile_sum( @@ -1194,6 +1320,26 @@ void FragmentMetadata::store_v15_or_higher( offset += nbytes; } + if (!dense_ && + version_ >= constants::fragment_metadata_global_order_bounds_version) { + const auto num_dims = array_schema_->dim_num(); + // Store global order mins + gt_offsets_.tile_global_order_min_offsets_.resize(num_dims); + for (unsigned i = 0; i < num_dims; ++i) { + gt_offsets_.tile_global_order_min_offsets_[i] = offset; + store_tile_global_order_mins(i, encryption_key, &nbytes); + offset += nbytes; + } + + // Store global order maxs + gt_offsets_.tile_global_order_max_offsets_.resize(num_dims); + for (unsigned i = 0; i < num_dims; ++i) { + gt_offsets_.tile_global_order_max_offsets_[i] = offset; + store_tile_global_order_maxs(i, encryption_key, &nbytes); + offset += nbytes; + } + } + // Store sums gt_offsets_.tile_sum_offsets_.resize(num); for (unsigned int i = 0; i < num; ++i) { @@ -1271,6 +1417,16 @@ void FragmentMetadata::set_num_tiles(uint64_t num_tiles) { if (array_schema_->is_nullable(it.first)) loaded_metadata_ptr_->tile_null_counts()[i].resize(num_tiles, 0); } + + // Sparse arrays also store the global order lower/upper bounds + if (!array_schema_->dense() && is_dim) { + const unsigned dimension = + array_schema_->domain().get_dimension_index(it.first); + loaded_metadata_ptr_->tile_global_order_min_buffer()[dimension].resize( + num_tiles * cell_size, 0); + loaded_metadata_ptr_->tile_global_order_max_buffer()[dimension].resize( + num_tiles * cell_size, 0); + } } if (!dense_) { @@ -2239,8 +2395,10 @@ void FragmentMetadata::load_generic_tile_offsets(Deserializer& deserializer) { load_generic_tile_offsets_v11(deserializer); } else if (version_ >= 12 && version_ < 16) { load_generic_tile_offsets_v12_v15(deserializer); + } else if (version_ >= 16 && version_ < 23) { + load_generic_tile_offsets_v16_v22(deserializer); } else { - load_generic_tile_offsets_v16_or_higher(deserializer); + load_generic_tile_offsets_v23_or_higher(deserializer); } } @@ -2394,7 +2552,7 @@ void FragmentMetadata::load_generic_tile_offsets_v12_v15( deserializer.read(); } -void FragmentMetadata::load_generic_tile_offsets_v16_or_higher( +void FragmentMetadata::load_generic_tile_offsets_v16_v22( Deserializer& deserializer) { // Load R-Tree offset gt_offsets_.rtree_ = deserializer.read(); @@ -2440,6 +2598,65 @@ void FragmentMetadata::load_generic_tile_offsets_v16_or_higher( gt_offsets_.processed_conditions_offsets_ = deserializer.read(); } +void FragmentMetadata::load_generic_tile_offsets_v23_or_higher( + Deserializer& deserializer) { + // Load R-Tree offset + gt_offsets_.rtree_ = deserializer.read(); + + // Load offsets for tile offsets + auto num = num_dims_and_attrs(); + gt_offsets_.tile_offsets_.resize(num); + deserializer.read(>_offsets_.tile_offsets_[0], num * sizeof(uint64_t)); + + // Load offsets for tile var offsets + gt_offsets_.tile_var_offsets_.resize(num); + deserializer.read(>_offsets_.tile_var_offsets_[0], num * sizeof(uint64_t)); + + // Load offsets for tile var sizes + gt_offsets_.tile_var_sizes_.resize(num); + deserializer.read(>_offsets_.tile_var_sizes_[0], num * sizeof(uint64_t)); + + // Load offsets for tile validity offsets + gt_offsets_.tile_validity_offsets_.resize(num); + deserializer.read( + >_offsets_.tile_validity_offsets_[0], num * sizeof(uint64_t)); + + // Load offsets for tile min offsets + gt_offsets_.tile_min_offsets_.resize(num); + deserializer.read(>_offsets_.tile_min_offsets_[0], num * sizeof(uint64_t)); + + // Load offsets for tile max offsets + gt_offsets_.tile_max_offsets_.resize(num); + deserializer.read(>_offsets_.tile_max_offsets_[0], num * sizeof(uint64_t)); + + if (!dense_) { + // Load offsets for the tile global order bounds + const auto num_dims = array_schema_->dim_num(); + gt_offsets_.tile_global_order_min_offsets_.resize(num_dims); + gt_offsets_.tile_global_order_max_offsets_.resize(num_dims); + deserializer.read( + gt_offsets_.tile_global_order_min_offsets_.data(), + num_dims * sizeof(uint64_t)); + deserializer.read( + gt_offsets_.tile_global_order_max_offsets_.data(), + num_dims * sizeof(uint64_t)); + } + + // Load offsets for tile sum offsets + gt_offsets_.tile_sum_offsets_.resize(num); + deserializer.read(>_offsets_.tile_sum_offsets_[0], num * sizeof(uint64_t)); + + // Load offsets for tile null count offsets + gt_offsets_.tile_null_count_offsets_.resize(num); + deserializer.read( + >_offsets_.tile_null_count_offsets_[0], num * sizeof(uint64_t)); + + gt_offsets_.fragment_min_max_sum_null_count_offset_ = + deserializer.read(); + + gt_offsets_.processed_conditions_offsets_ = deserializer.read(); +} + void FragmentMetadata::load_array_schema_name(Deserializer& deserializer) { uint64_t size = 0; size = deserializer.read(); @@ -2666,6 +2883,18 @@ void FragmentMetadata::write_generic_tile_offsets( serializer.write(>_offsets_.tile_max_offsets_[0], num * sizeof(uint64_t)); } + if (!dense_ && + version_ >= constants::fragment_metadata_global_order_bounds_version) { + // Write the tile global order bound offsets + const auto num_dims = array_schema_->dim_num(); + serializer.write( + gt_offsets_.tile_global_order_min_offsets_.data(), + num_dims * sizeof(uint64_t)); + serializer.write( + gt_offsets_.tile_global_order_max_offsets_.data(), + num_dims * sizeof(uint64_t)); + } + // Write tile sum offsets if (version_ >= 11) { serializer.write(>_offsets_.tile_sum_offsets_[0], num * sizeof(uint64_t)); @@ -3030,6 +3259,70 @@ void FragmentMetadata::write_tile_maxs(unsigned idx, Serializer& serializer) { } } +void FragmentMetadata::store_tile_global_order_mins( + unsigned dim, const EncryptionKey& encryption_key, uint64_t* nbytes) { + SizeComputationSerializer size_computation_serializer; + write_tile_global_order_mins(dim, size_computation_serializer); + + auto tile{WriterTile::from_generic( + size_computation_serializer.size(), memory_tracker_)}; + + Serializer serializer(tile->data(), tile->size()); + write_tile_global_order_mins(dim, serializer); + write_generic_tile_to_file(encryption_key, tile, nbytes); + + resources_->stats().add_counter("write_global_order_mins_size", *nbytes); +} + +void FragmentMetadata::write_tile_global_order_mins( + unsigned dim, Serializer& serializer) { + const auto& fixedPart = + loaded_metadata_ptr_->tile_global_order_min_buffer()[dim]; + const auto& varPart = + loaded_metadata_ptr_->tile_global_order_min_var_buffer()[dim]; + serializer.write(fixedPart.size()); + serializer.write(varPart.size()); + + if (!fixedPart.empty()) { + serializer.write(fixedPart.data(), fixedPart.size()); + } + if (!varPart.empty()) { + serializer.write(varPart.data(), varPart.size()); + } +} + +void FragmentMetadata::store_tile_global_order_maxs( + unsigned dim, const EncryptionKey& encryption_key, uint64_t* nbytes) { + SizeComputationSerializer size_computation_serializer; + write_tile_global_order_maxs(dim, size_computation_serializer); + + auto tile{WriterTile::from_generic( + size_computation_serializer.size(), memory_tracker_)}; + + Serializer serializer(tile->data(), tile->size()); + write_tile_global_order_maxs(dim, serializer); + write_generic_tile_to_file(encryption_key, tile, nbytes); + + resources_->stats().add_counter("write_global_order_maxs_size", *nbytes); +} + +void FragmentMetadata::write_tile_global_order_maxs( + unsigned dim, Serializer& serializer) { + const auto& fixedPart = + loaded_metadata_ptr_->tile_global_order_max_buffer()[dim]; + const auto& varPart = + loaded_metadata_ptr_->tile_global_order_max_var_buffer()[dim]; + serializer.write(fixedPart.size()); + serializer.write(varPart.size()); + + if (!fixedPart.empty()) { + serializer.write(fixedPart.data(), fixedPart.size()); + } + if (!varPart.empty()) { + serializer.write(varPart.data(), varPart.size()); + } +} + void FragmentMetadata::store_tile_sums( unsigned idx, const EncryptionKey& encryption_key, uint64_t* nbytes) { SizeComputationSerializer size_computation_serializer; diff --git a/tiledb/sm/fragment/fragment_metadata.h b/tiledb/sm/fragment/fragment_metadata.h index fe2e6e69a35..4553bc6080a 100644 --- a/tiledb/sm/fragment/fragment_metadata.h +++ b/tiledb/sm/fragment/fragment_metadata.h @@ -63,6 +63,7 @@ class Buffer; class EncryptionKey; class TileMetadata; class MemoryTracker; +class WriterTileTuple; class FragmentMetadataStatusException : public StatusException { public: @@ -139,6 +140,8 @@ class FragmentMetadata { std::vector tile_validity_offsets_; std::vector tile_min_offsets_; std::vector tile_max_offsets_; + std::vector tile_global_order_min_offsets_; // per dimension only + std::vector tile_global_order_max_offsets_; // per dimension only std::vector tile_sum_offsets_; std::vector tile_null_count_offsets_; uint64_t fragment_min_max_sum_null_count_offset_; @@ -596,6 +599,26 @@ class FragmentMetadata { */ void convert_tile_min_max_var_sizes_to_offsets(const std::string& name); + /** + * Populate fixed parts of bounds for a dimension and tile using the + * coordinate values in `data`. + */ + void set_tile_global_order_bounds_fixed( + const std::string& dim_name, uint64_t tile, const WriterTileTuple& data); + + /** + * Populate var parts of bounds for a dimension and tile using the coordinate + * values in `data`. + */ + void set_tile_global_order_bounds_var( + const std::string& dim_name, uint64_t tile, const WriterTileTuple& data); + + /** + * Converts global order min/max sizes to offsets. + */ + void convert_tile_global_order_bounds_sizes_to_offsets( + const std::string& dim_name); + /** * Sets a tile sum for the input attribute. * @@ -1030,9 +1053,15 @@ class FragmentMetadata { /** * Loads the generic tile offsets from the buffer. Applicable to - * versions 16 or higher. + * versions 16 to 22. */ - void load_generic_tile_offsets_v16_or_higher(Deserializer& deserializer); + void load_generic_tile_offsets_v16_v22(Deserializer& deserializer); + + /** + * Loads the generic tile offsets from the buffer. Applicable to + * versions 23 or higher. + */ + void load_generic_tile_offsets_v23_or_higher(Deserializer& deserializer); /** * Loads the array schema name. @@ -1312,6 +1341,36 @@ class FragmentMetadata { */ void write_tile_maxs(unsigned idx, Serializer& serializer); + /** + * Writes the global order minimum of each tile to storage. + * + * @param dim The index of the dimension + * @param encryption_key The encryption key + * @param nbytes[in/out] The total number of bytes written + */ + void store_tile_global_order_mins( + unsigned dim, const EncryptionKey& encryption_key, uint64_t* nbytes); + + /** + * Writes the global order minimum of each tile to the input buffer. + */ + void write_tile_global_order_mins(unsigned dim, Serializer& serializer); + + /** + * Writes the global order maximum of each tile to storage. + * + * @param dim The index of the dimension + * @param encryption_key The encryption key + * @param nbytes[in/out] The total number of bytes written + */ + void store_tile_global_order_maxs( + unsigned dim, const EncryptionKey& encryption_key, uint64_t* nbytes); + + /** + * Writes the global order maximum of each tile to the input buffer. + */ + void write_tile_global_order_maxs(unsigned dim, Serializer& serializer); + /** * Writes the sums of the input attribute to storage. * diff --git a/tiledb/sm/fragment/loaded_fragment_metadata.cc b/tiledb/sm/fragment/loaded_fragment_metadata.cc index fbeb356f340..ddc29ed2fd6 100644 --- a/tiledb/sm/fragment/loaded_fragment_metadata.cc +++ b/tiledb/sm/fragment/loaded_fragment_metadata.cc @@ -67,6 +67,14 @@ LoadedFragmentMetadata::LoadedFragmentMetadata( , tile_max_buffer_(memory_tracker_->get_resource(MemoryType::TILE_MAX_VALS)) , tile_max_var_buffer_( memory_tracker_->get_resource(MemoryType::TILE_MAX_VALS)) + , tile_global_order_min_buffer_( + memory_tracker_->get_resource(MemoryType::TILE_GLOBAL_ORDER_MIN_VALS)) + , tile_global_order_max_buffer_( + memory_tracker_->get_resource(MemoryType::TILE_GLOBAL_ORDER_MAX_VALS)) + , tile_global_order_min_var_buffer_( + memory_tracker_->get_resource(MemoryType::TILE_GLOBAL_ORDER_MIN_VALS)) + , tile_global_order_max_var_buffer_( + memory_tracker_->get_resource(MemoryType::TILE_GLOBAL_ORDER_MAX_VALS)) , tile_sums_(memory_tracker_->get_resource(MemoryType::TILE_SUMS)) , tile_null_counts_( memory_tracker_->get_resource(MemoryType::TILE_NULL_COUNTS)) { @@ -221,6 +229,10 @@ void LoadedFragmentMetadata::resize_offsets(uint64_t size) { tile_min_var_buffer().resize(size); tile_max_buffer().resize(size); tile_max_var_buffer().resize(size); + tile_global_order_min_buffer().resize(size); + tile_global_order_min_var_buffer().resize(size); + tile_global_order_max_buffer().resize(size); + tile_global_order_max_var_buffer().resize(size); tile_sums().resize(size); tile_null_counts().resize(size); fragment_mins_.resize(size); @@ -233,6 +245,8 @@ void LoadedFragmentMetadata::resize_offsets(uint64_t size) { loaded_metadata_.tile_validity_offsets_.resize(size, false); loaded_metadata_.tile_min_.resize(size, false); loaded_metadata_.tile_max_.resize(size, false); + loaded_metadata_.tile_global_order_min_.resize(size, false); + loaded_metadata_.tile_global_order_max_.resize(size, false); loaded_metadata_.tile_sum_.resize(size, false); loaded_metadata_.tile_null_count_.resize(size, false); } diff --git a/tiledb/sm/fragment/loaded_fragment_metadata.h b/tiledb/sm/fragment/loaded_fragment_metadata.h index cd34362d73b..2580342c131 100644 --- a/tiledb/sm/fragment/loaded_fragment_metadata.h +++ b/tiledb/sm/fragment/loaded_fragment_metadata.h @@ -60,6 +60,8 @@ class LoadedFragmentMetadata { std::vector tile_validity_offsets_; std::vector tile_min_; std::vector tile_max_; + std::vector tile_global_order_min_; + std::vector tile_global_order_max_; std::vector tile_sum_; std::vector tile_null_count_; bool fragment_min_max_sum_null_count_ = false; @@ -332,6 +334,14 @@ class LoadedFragmentMetadata { virtual void load_fragment_min_max_sum_null_count( const EncryptionKey& encryption_key) = 0; + /** + * Loads the tile global order bounds for the fragment. + * + * @param encrpytion_key The key the array was opened with. + */ + virtual void load_fragment_tile_global_order_bounds( + const EncryptionKey& encryption_key) = 0; + /** * Loads the processed conditions for the fragment. The processed conditions * is the list of delete/update conditions that have already been applied for @@ -456,6 +466,38 @@ class LoadedFragmentMetadata { return tile_max_var_buffer_; } + inline const auto& tile_global_order_min_buffer() const { + return tile_global_order_min_buffer_; + } + + inline const auto& tile_global_order_min_var_buffer() const { + return tile_global_order_min_var_buffer_; + } + + inline const auto& tile_global_order_max_buffer() const { + return tile_global_order_max_buffer_; + } + + inline const auto& tile_global_order_max_var_buffer() const { + return tile_global_order_max_var_buffer_; + } + + inline auto& tile_global_order_min_buffer() { + return tile_global_order_min_buffer_; + } + + inline auto& tile_global_order_min_var_buffer() { + return tile_global_order_min_var_buffer_; + } + + inline auto& tile_global_order_max_buffer() { + return tile_global_order_max_buffer_; + } + + inline auto& tile_global_order_max_var_buffer() { + return tile_global_order_max_var_buffer_; + } + /** Returns an RTree for the MBRs. */ inline const RTree& rtree() const { return rtree_; @@ -655,6 +697,40 @@ class LoadedFragmentMetadata { */ tdb::pmr::vector> tile_max_var_buffer_; + /** + * The tile global order minima. + * + * The outer vector is indexed by dimension `d`. + * The inner vector is the concatenated values of the `d`th + * dimension of the global order minimum coordinate for each tile. + * + * For variable-length dimensions the value stored here is the offset + * into the corresponding variable-length buffer. + */ + tdb::pmr::vector> tile_global_order_min_buffer_; + + /** + * The tile global order maxima. + * + * The outer vector is indexed by dimension `d`. + * The inner vector is the concatenated values of the `d`th + * dimension of the global order maximum coordinate for each tile. + * + * For variable-length dimensions the value stored here is the offset + * into the corresponding variable-length buffer. + */ + tdb::pmr::vector> tile_global_order_max_buffer_; + + /** + * The tile global order minima, variable-length part. + */ + tdb::pmr::vector> tile_global_order_min_var_buffer_; + + /** + * The tile global order maxima, variable-length part. + */ + tdb::pmr::vector> tile_global_order_max_var_buffer_; + /** * The tile sum values, ignored for var sized attributes/dimensions. */ @@ -765,6 +841,24 @@ class LoadedFragmentMetadata { virtual void load_tile_max_values( const EncryptionKey& encryption_key, unsigned idx) = 0; + /** + * Loads the global order minimum values for the given dimension from storage. + * + * @param encryption_key The encryption key + * @param dimension Dimension index + */ + virtual void load_tile_global_order_min_values( + const EncryptionKey& encryption_key, unsigned dimension) = 0; + + /** + * Loads the global order minimum values for the given dimension from storage. + * + * @param encryption_key The encryption key + * @param dimension Dimension index + */ + virtual void load_tile_global_order_max_values( + const EncryptionKey& encryption_key, unsigned dimension) = 0; + /** * Loads the sum values for the input attribute idx from storage. * diff --git a/tiledb/sm/fragment/ondemand_fragment_metadata.cc b/tiledb/sm/fragment/ondemand_fragment_metadata.cc index 45f070ad4dd..d07dccb5a33 100644 --- a/tiledb/sm/fragment/ondemand_fragment_metadata.cc +++ b/tiledb/sm/fragment/ondemand_fragment_metadata.cc @@ -113,6 +113,19 @@ void OndemandFragmentMetadata::load_fragment_min_max_sum_null_count( loaded_metadata_.fragment_min_max_sum_null_count_ = true; } +void OndemandFragmentMetadata::load_fragment_tile_global_order_bounds( + const EncryptionKey& encryption_key) { + if (parent_fragment_.version_ < + constants::fragment_metadata_global_order_bounds_version) { + return; + } + + for (unsigned d = 0; d < parent_fragment_.array_schema_->dim_num(); d++) { + load_tile_global_order_min_values(encryption_key, d); + load_tile_global_order_max_values(encryption_key, d); + } +} + void OndemandFragmentMetadata::load_processed_conditions( const EncryptionKey& encryption_key) { if (loaded_metadata_.processed_conditions_) { @@ -307,18 +320,12 @@ void OndemandFragmentMetadata::load_tile_validity_offsets( } } -// ===== FORMAT ===== -// tile_min_values#0_size_buffer (uint64_t) -// tile_min_values#0_size_buffer_var (uint64_t) -// tile_min_values#0_buffer -// tile_min_values#0_buffer_var -// ... -// tile_min_values#_size_buffer (uint64_t) -// tile_min_values#_size_buffer_var (uint64_t) -// tile_min_values#_buffer -// tile_min_values#_buffer_var -void OndemandFragmentMetadata::load_tile_min_values( - unsigned idx, Deserializer& deserializer) { +static void load_tile_bound_values( + auto& fixed_buffer, + auto& var_buffer, + MemoryTracker* memory_tracker, + MemoryType mem_type, + Deserializer& deserializer) { uint64_t buffer_size = 0; uint64_t var_buffer_size = 0; @@ -331,26 +338,47 @@ void OndemandFragmentMetadata::load_tile_min_values( // Get tile mins if (buffer_size != 0) { auto size = buffer_size + var_buffer_size; - if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory(size, MemoryType::TILE_MIN_VALS)) { + if (memory_tracker != nullptr && + !memory_tracker->take_memory(size, mem_type)) { throw FragmentMetadataStatusException( - "Cannot load min values; Insufficient memory budget; Needed " + - std::to_string(size) + " but only had " + - std::to_string(memory_tracker_->get_memory_available()) + + "Cannot load " + memory_type_to_str(mem_type) + + "; Insufficient memory budget; Needed " + std::to_string(size) + + " but only had " + + std::to_string(memory_tracker->get_memory_available()) + " from budget " + - std::to_string(memory_tracker_->get_memory_budget())); + std::to_string(memory_tracker->get_memory_budget())); } - tile_min_buffer_[idx].resize(buffer_size); - deserializer.read(&tile_min_buffer_[idx][0], buffer_size); + fixed_buffer.resize(buffer_size); + deserializer.read(&fixed_buffer[0], buffer_size); if (var_buffer_size) { - tile_min_var_buffer_[idx].resize(var_buffer_size); - deserializer.read(&tile_min_var_buffer_[idx][0], var_buffer_size); + var_buffer.resize(var_buffer_size); + deserializer.read(&var_buffer[0], var_buffer_size); } } } +// ===== FORMAT ===== +// tile_min_values#0_size_buffer (uint64_t) +// tile_min_values#0_size_buffer_var (uint64_t) +// tile_min_values#0_buffer +// tile_min_values#0_buffer_var +// ... +// tile_min_values#_size_buffer (uint64_t) +// tile_min_values#_size_buffer_var (uint64_t) +// tile_min_values#_buffer +// tile_min_values#_buffer_var +void OndemandFragmentMetadata::load_tile_min_values( + unsigned idx, Deserializer& deserializer) { + load_tile_bound_values( + tile_min_buffer_[idx], + tile_min_var_buffer_[idx], + memory_tracker_.get(), + MemoryType::TILE_MIN_VALS, + deserializer); +} + // ===== FORMAT ===== // tile_max_values#0_size_buffer (uint64_t) // tile_max_values#0_size_buffer_var (uint64_t) @@ -363,36 +391,52 @@ void OndemandFragmentMetadata::load_tile_min_values( // tile_max_values#_buffer_var void OndemandFragmentMetadata::load_tile_max_values( unsigned idx, Deserializer& deserializer) { - uint64_t buffer_size = 0; - uint64_t var_buffer_size = 0; - - // Get buffer size - buffer_size = deserializer.read(); - - // Get var buffer size - var_buffer_size = deserializer.read(); - - // Get tile maxs - if (buffer_size != 0) { - auto size = buffer_size + var_buffer_size; - if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory(size, MemoryType::TILE_MAX_VALS)) { - throw FragmentMetadataStatusException( - "Cannot load max values; Insufficient memory budget; Needed " + - std::to_string(size) + " but only had " + - std::to_string(memory_tracker_->get_memory_available()) + - " from budget " + - std::to_string(memory_tracker_->get_memory_budget())); - } + load_tile_bound_values( + tile_max_buffer_[idx], + tile_max_var_buffer_[idx], + memory_tracker_.get(), + MemoryType::TILE_MAX_VALS, + deserializer); +} - tile_max_buffer_[idx].resize(buffer_size); - deserializer.read(&tile_max_buffer_[idx][0], buffer_size); +// ===== FORMAT ===== +// tile_global_order_min_values#0_size_buffer (uint64_t) +// tile_global_order_min_values#0_size_buffer_var (uint64_t) +// tile_global_order_min_values#0_buffer +// tile_global_order_min_values#0_buffer_var +// ... +// tile_global_order_min_values#_size_buffer (uint64_t) +// tile_global_order_min_values#_size_buffer_var (uint64_t) +// tile_global_order_min_values#_buffer +// tile_global_order_min_values#_buffer_var +void OndemandFragmentMetadata::load_tile_global_order_min_values( + unsigned dimension, Deserializer& deserializer) { + load_tile_bound_values( + tile_global_order_min_buffer_[dimension], + tile_global_order_min_var_buffer_[dimension], + memory_tracker_.get(), + MemoryType::TILE_GLOBAL_ORDER_MIN_VALS, + deserializer); +} - if (var_buffer_size) { - tile_max_var_buffer_[idx].resize(var_buffer_size); - deserializer.read(&tile_max_var_buffer_[idx][0], var_buffer_size); - } - } +// ===== FORMAT ===== +// tile_global_order_max_values#0_size_buffer (uint64_t) +// tile_global_order_max_values#0_size_buffer_var (uint64_t) +// tile_global_order_max_values#0_buffer +// tile_global_order_max_values#0_buffer_var +// ... +// tile_global_order_max_values#_size_buffer (uint64_t) +// tile_global_order_max_values#_size_buffer_var (uint64_t) +// tile_global_order_max_values#_buffer +// tile_global_order_max_values#_buffer_var +void OndemandFragmentMetadata::load_tile_global_order_max_values( + unsigned dimension, Deserializer& deserializer) { + load_tile_bound_values( + tile_global_order_max_buffer_[dimension], + tile_global_order_max_var_buffer_[dimension], + memory_tracker_.get(), + MemoryType::TILE_GLOBAL_ORDER_MAX_VALS, + deserializer); } // ===== FORMAT ===== @@ -577,6 +621,56 @@ void OndemandFragmentMetadata::load_tile_max_values( loaded_metadata_.tile_max_[idx] = true; } +void OndemandFragmentMetadata::load_tile_global_order_min_values( + const EncryptionKey& encryption_key, unsigned dimension) { + if (parent_fragment_.version_ < + constants::fragment_metadata_global_order_bounds_version) { + return; + } + + std::lock_guard lock(parent_fragment_.mtx_); + + if (loaded_metadata_.tile_global_order_min_[dimension]) { + return; + } + + auto tile = parent_fragment_.read_generic_tile_from_file( + encryption_key, + parent_fragment_.gt_offsets_.tile_global_order_min_offsets_[dimension]); + parent_fragment_.resources_->stats().add_counter( + "read_tile_global_order_min_size", tile->size()); + + Deserializer deserializer(tile->data(), tile->size()); + load_tile_global_order_min_values(dimension, deserializer); + + loaded_metadata_.tile_global_order_min_[dimension] = true; +} + +void OndemandFragmentMetadata::load_tile_global_order_max_values( + const EncryptionKey& encryption_key, unsigned dimension) { + if (parent_fragment_.version_ < + constants::fragment_metadata_global_order_bounds_version) { + return; + } + + std::lock_guard lock(parent_fragment_.mtx_); + + if (loaded_metadata_.tile_global_order_max_[dimension]) { + return; + } + + auto tile = parent_fragment_.read_generic_tile_from_file( + encryption_key, + parent_fragment_.gt_offsets_.tile_global_order_max_offsets_[dimension]); + parent_fragment_.resources_->stats().add_counter( + "read_tile_global_order_max_size", tile->size()); + + Deserializer deserializer(tile->data(), tile->size()); + load_tile_global_order_max_values(dimension, deserializer); + + loaded_metadata_.tile_global_order_max_[dimension] = true; +} + void OndemandFragmentMetadata::load_tile_sum_values( const EncryptionKey& encryption_key, unsigned idx) { if (parent_fragment_.version_ < constants::tile_metadata_min_version) { diff --git a/tiledb/sm/fragment/ondemand_fragment_metadata.h b/tiledb/sm/fragment/ondemand_fragment_metadata.h index 651741b7be3..e8573c88a0c 100644 --- a/tiledb/sm/fragment/ondemand_fragment_metadata.h +++ b/tiledb/sm/fragment/ondemand_fragment_metadata.h @@ -75,6 +75,14 @@ class OndemandFragmentMetadata : public LoadedFragmentMetadata { virtual void load_fragment_min_max_sum_null_count( const EncryptionKey& encryption_key) override; + /** + * Loads the tile global order bounds for the fragment. + * + * @param encrpytion_key The key the array was opened with. + */ + virtual void load_fragment_tile_global_order_bounds( + const EncryptionKey& encryption_key) override; + /** * Loads the processed conditions for the fragment. The processed conditions * is the list of delete/update conditions that have already been applied for @@ -223,6 +231,32 @@ class OndemandFragmentMetadata : public LoadedFragmentMetadata { virtual void load_tile_max_values( const EncryptionKey& encryption_key, unsigned idx) override; + /** + * Loads the tile global order minima for the dimension `dimension`. + */ + void load_tile_global_order_min_values( + unsigned dimension, Deserializer& deserializer); + + /** + * Loads the tile global order minima for the dimension `dimension` from + * storage. + */ + void load_tile_global_order_min_values( + const EncryptionKey& encryption_key, unsigned dimension) override; + + /** + * Loads the tile global order maxima for the dimension `dimension`. + */ + void load_tile_global_order_max_values( + unsigned dimension, Deserializer& deserializer); + + /** + * Loads the tile global order maxima for the dimension `dimension` from + * storage. + */ + void load_tile_global_order_max_values( + const EncryptionKey& encryption_key, unsigned dimension) override; + /** * Loads the sum values for the input attribute from the input buffer. * diff --git a/tiledb/sm/fragment/v1v2preloaded_fragment_metadata.cc b/tiledb/sm/fragment/v1v2preloaded_fragment_metadata.cc index 5156316161c..eb6b60e5833 100644 --- a/tiledb/sm/fragment/v1v2preloaded_fragment_metadata.cc +++ b/tiledb/sm/fragment/v1v2preloaded_fragment_metadata.cc @@ -165,6 +165,12 @@ void V1V2PreloadedFragmentMetadata::load_fragment_min_max_sum_null_count( return; } +void V1V2PreloadedFragmentMetadata::load_fragment_tile_global_order_bounds( + const EncryptionKey&) { + // N/A for v1_v2 preloaded meta + return; +} + /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ @@ -205,6 +211,18 @@ void V1V2PreloadedFragmentMetadata::load_tile_max_values( return; } +void V1V2PreloadedFragmentMetadata::load_tile_global_order_min_values( + const EncryptionKey&, unsigned) { + // N/A for v1_v2 preloaded meta + return; +} + +void V1V2PreloadedFragmentMetadata::load_tile_global_order_max_values( + const EncryptionKey&, unsigned) { + // N/A for v1_v2 preloaded meta + return; +} + void V1V2PreloadedFragmentMetadata::load_tile_sum_values( const EncryptionKey&, unsigned) { // N/A for v1_v2 preloaded meta diff --git a/tiledb/sm/fragment/v1v2preloaded_fragment_metadata.h b/tiledb/sm/fragment/v1v2preloaded_fragment_metadata.h index 7c1e8e5e6bf..437a6da06b3 100644 --- a/tiledb/sm/fragment/v1v2preloaded_fragment_metadata.h +++ b/tiledb/sm/fragment/v1v2preloaded_fragment_metadata.h @@ -105,6 +105,14 @@ class V1V2PreloadedFragmentMetadata : public LoadedFragmentMetadata { virtual void load_fragment_min_max_sum_null_count( const EncryptionKey& encryption_key) override; + /** + * Loads the tile global order bounds for the fragment. + * + * @param encrpytion_key The key the array was opened with. + */ + virtual void load_fragment_tile_global_order_bounds( + const EncryptionKey& encryption_key) override; + /** * Loads the processed conditions for the fragment. The processed conditions * is the list of delete/update conditions that have already been applied for @@ -201,6 +209,30 @@ class V1V2PreloadedFragmentMetadata : public LoadedFragmentMetadata { virtual void load_tile_max_values( const EncryptionKey& encryption_key, unsigned idx) override; + /** + * Loads the global order minimum values for the given dimension from storage. + * + * This is a no-op because this field does not exist in format versions + * using this class. + * + * @param encryption_key The encryption key + * @param dimension Dimension index + */ + virtual void load_tile_global_order_min_values( + const EncryptionKey& encryption_key, unsigned dimension) override; + + /** + * Loads the global order maximum values for the given dimension from storage. + * + * This is a no-op because this field does not exist in format versions + * using this class. + * + * @param encryption_key The encryption key + * @param dimension Dimension index + */ + virtual void load_tile_global_order_max_values( + const EncryptionKey& encryption_key, unsigned dimension) override; + /** * Loads the sum values for the input attribute idx from storage. * diff --git a/tiledb/sm/misc/constants.cc b/tiledb/sm/misc/constants.cc index 18c20eacdd0..44720f33962 100644 --- a/tiledb/sm/misc/constants.cc +++ b/tiledb/sm/misc/constants.cc @@ -690,7 +690,7 @@ const int32_t library_version[3] = { TILEDB_VERSION_MAJOR, TILEDB_VERSION_MINOR, TILEDB_VERSION_PATCH}; /** The TileDB serialization base format version number. */ -const format_version_t format_version = 22; +const format_version_t format_version = 23; /** The lowest version supported for back compat writes. */ const format_version_t back_compat_writes_min_format_version = 7; @@ -722,6 +722,10 @@ const format_version_t current_domain_version = 0; /** The NDRectangle current domain */ const std::string current_domain_ndrectangle_str = "NDRECTANGLE"; +/** The lowest version where fragment metadata contains per-tile global order + * bounds */ +const format_version_t fragment_metadata_global_order_bounds_version = 23; + /** The maximum size of a tile chunk (unit of compression) in bytes. */ const uint64_t max_tile_chunk_size = 64 * 1024; diff --git a/tiledb/sm/misc/constants.h b/tiledb/sm/misc/constants.h index 6db29c26702..9e6561894f5 100644 --- a/tiledb/sm/misc/constants.h +++ b/tiledb/sm/misc/constants.h @@ -717,6 +717,10 @@ extern const format_version_t current_domain_version; /** The NDRectangle current_domain */ extern const std::string current_domain_ndrectangle_str; +/** The lowest version where fragment metadata contains per-tile global order + * bounds */ +extern const format_version_t fragment_metadata_global_order_bounds_version; + /** The maximum size of a tile chunk (unit of compression) in bytes. */ extern const uint64_t max_tile_chunk_size; diff --git a/tiledb/sm/query/writers/writer_base.cc b/tiledb/sm/query/writers/writer_base.cc index e0fcd5683c0..2bfd1223c56 100644 --- a/tiledb/sm/query/writers/writer_base.cc +++ b/tiledb/sm/query/writers/writer_base.cc @@ -1068,6 +1068,16 @@ Status WriterBase::write_tiles( attr, idx - start_tile_idx, tiles[idx].max()); } } + + if (array_schema_.is_dim(attr)) { + if (array_schema_.var_size(attr)) { + frag_meta->convert_tile_global_order_bounds_sizes_to_offsets(attr); + } + for (uint64_t idx = start_tile_idx; idx < end_tile_idx; idx++) { + frag_meta->set_tile_global_order_bounds_var( + attr, idx - start_tile_idx, tiles[idx]); + } + } return Status::Ok(); })); } @@ -1137,11 +1147,17 @@ Status WriterBase::write_tiles( frag_meta->set_tile_min_var_size(name, tile_id, tile.min().size()); frag_meta->set_tile_max_var_size(name, tile_id, tile.max().size()); } + if (array_schema_.is_dim(name)) { + frag_meta->set_tile_global_order_bounds_fixed(name, tile_id, tile); + } } else { if (has_min_max_md && null_count != frag_meta->cell_num(tile_id)) { frag_meta->set_tile_min(name, tile_id, tile.min()); frag_meta->set_tile_max(name, tile_id, tile.max()); } + if (array_schema_.is_dim(name)) { + frag_meta->set_tile_global_order_bounds_fixed(name, tile_id, tile); + } if (has_sum_md) { frag_meta->set_tile_sum(name, tile_id, tile.sum()); diff --git a/tiledb/sm/serialization/fragment_metadata.cc b/tiledb/sm/serialization/fragment_metadata.cc index bd4ebb3e372..f2f91834881 100644 --- a/tiledb/sm/serialization/fragment_metadata.cc +++ b/tiledb/sm/serialization/fragment_metadata.cc @@ -100,6 +100,24 @@ void generic_tile_offsets_from_capnp( gt_offsets.tile_max_offsets_.emplace_back(tile_max_offset); } } + if (gt_reader.hasTileGlobalOrderMinOffsets()) { + auto tile_global_order_min_offsets = + gt_reader.getTileGlobalOrderMinOffsets(); + gt_offsets.tile_global_order_min_offsets_.reserve( + tile_global_order_min_offsets.size()); + for (const auto& offset : tile_global_order_min_offsets) { + gt_offsets.tile_global_order_min_offsets_.emplace_back(offset); + } + } + if (gt_reader.hasTileGlobalOrderMaxOffsets()) { + auto tile_global_order_max_offsets = + gt_reader.getTileGlobalOrderMaxOffsets(); + gt_offsets.tile_global_order_max_offsets_.reserve( + tile_global_order_max_offsets.size()); + for (const auto& offset : tile_global_order_max_offsets) { + gt_offsets.tile_global_order_max_offsets_.emplace_back(offset); + } + } if (gt_reader.hasTileSumOffsets()) { auto tile_sum_offsets = gt_reader.getTileSumOffsets(); gt_offsets.tile_sum_offsets_.reserve(tile_sum_offsets.size()); @@ -120,6 +138,16 @@ void generic_tile_offsets_from_capnp( gt_reader.getProcessedConditionsOffsets(); } +static void tile_bounds_from_capnp(const auto& reader, auto& buffers) { + for (const auto& t : reader) { + auto& last = buffers.emplace_back(); + last.reserve(t.size()); + for (const auto& v : t) { + last.emplace_back(v); + } + } +} + Status fragment_metadata_from_capnp( const shared_ptr& fragment_array_schema, const capnp::FragmentMetadata::Reader& frag_meta_reader, @@ -258,49 +286,47 @@ Status fragment_metadata_from_capnp( } if (frag_meta_reader.hasTileMinBuffer()) { auto tileminbuffer_reader = frag_meta_reader.getTileMinBuffer(); - for (const auto& t : tileminbuffer_reader) { - auto& last = - frag_meta->loaded_metadata()->tile_min_buffer().emplace_back(); - last.reserve(t.size()); - for (const auto& v : t) { - last.emplace_back(v); - } - } + tile_bounds_from_capnp( + tileminbuffer_reader, frag_meta->loaded_metadata()->tile_min_buffer()); loaded_metadata.tile_min_.resize(tileminbuffer_reader.size(), false); } if (frag_meta_reader.hasTileMinVarBuffer()) { - auto tileminvarbuffer_reader = frag_meta_reader.getTileMinVarBuffer(); - for (const auto& t : tileminvarbuffer_reader) { - auto& last = - frag_meta->loaded_metadata()->tile_min_var_buffer().emplace_back(); - last.reserve(t.size()); - for (const auto& v : t) { - last.emplace_back(v); - } - } + tile_bounds_from_capnp( + frag_meta_reader.getTileMinVarBuffer(), + frag_meta->loaded_metadata()->tile_min_var_buffer()); } if (frag_meta_reader.hasTileMaxBuffer()) { auto tilemaxbuffer_reader = frag_meta_reader.getTileMaxBuffer(); - for (const auto& t : tilemaxbuffer_reader) { - auto& last = - frag_meta->loaded_metadata()->tile_max_buffer().emplace_back(); - last.reserve(t.size()); - for (const auto& v : t) { - last.emplace_back(v); - } - } + tile_bounds_from_capnp( + tilemaxbuffer_reader, frag_meta->loaded_metadata()->tile_max_buffer()); loaded_metadata.tile_max_.resize(tilemaxbuffer_reader.size(), false); } if (frag_meta_reader.hasTileMaxVarBuffer()) { - auto tilemaxvarbuffer_reader = frag_meta_reader.getTileMaxVarBuffer(); - for (const auto& t : tilemaxvarbuffer_reader) { - auto& last = - frag_meta->loaded_metadata()->tile_max_var_buffer().emplace_back(); - last.reserve(t.size()); - for (const auto& v : t) { - last.emplace_back(v); - } - } + tile_bounds_from_capnp( + frag_meta_reader.getTileMaxVarBuffer(), + frag_meta->loaded_metadata()->tile_max_var_buffer()); + } + if (frag_meta_reader.hasTileGlobalOrderMinBuffer()) { + auto reader = frag_meta_reader.getTileGlobalOrderMinBuffer(); + tile_bounds_from_capnp( + reader, frag_meta->loaded_metadata()->tile_global_order_min_buffer()); + loaded_metadata.tile_global_order_min_.resize(reader.size(), false); + } + if (frag_meta_reader.hasTileGlobalOrderMinVarBuffer()) { + tile_bounds_from_capnp( + frag_meta_reader.getTileGlobalOrderMinVarBuffer(), + frag_meta->loaded_metadata()->tile_global_order_min_var_buffer()); + } + if (frag_meta_reader.hasTileGlobalOrderMaxBuffer()) { + auto reader = frag_meta_reader.getTileGlobalOrderMaxBuffer(); + tile_bounds_from_capnp( + reader, frag_meta->loaded_metadata()->tile_global_order_max_buffer()); + loaded_metadata.tile_global_order_max_.resize(reader.size(), false); + } + if (frag_meta_reader.hasTileGlobalOrderMaxVarBuffer()) { + tile_bounds_from_capnp( + frag_meta_reader.getTileGlobalOrderMaxVarBuffer(), + frag_meta->loaded_metadata()->tile_global_order_max_var_buffer()); } if (frag_meta_reader.hasTileSums()) { auto tilesums_reader = frag_meta_reader.getTileSums(); @@ -467,6 +493,24 @@ void generic_tile_offsets_to_capnp( builder.set(i, gt_tile_max_offsets[i]); } } + auto& gt_tile_global_order_min_offsets = + gt_offsets.tile_global_order_min_offsets_; + if (!gt_tile_global_order_min_offsets.empty()) { + auto builder = gt_offsets_builder.initTileGlobalOrderMinOffsets( + gt_tile_min_offsets.size()); + for (uint64_t i = 0; i < gt_tile_global_order_min_offsets.size(); ++i) { + builder.set(i, gt_tile_global_order_min_offsets[i]); + } + } + auto& gt_tile_global_order_max_offsets = + gt_offsets.tile_global_order_max_offsets_; + if (!gt_tile_global_order_max_offsets.empty()) { + auto builder = gt_offsets_builder.initTileGlobalOrderMaxOffsets( + gt_tile_max_offsets.size()); + for (uint64_t i = 0; i < gt_tile_global_order_max_offsets.size(); ++i) { + builder.set(i, gt_tile_global_order_max_offsets[i]); + } + } auto& gt_tile_sum_offsets = gt_offsets.tile_sum_offsets_; if (!gt_tile_sum_offsets.empty()) { auto builder = @@ -537,6 +581,32 @@ void fragment_meta_sizes_offsets_to_capnp( } } +static void tile_bounds_to_capnp( + const auto& fixedBuffers, + const auto& varBuffers, + auto initFixedBuilder, + auto initVarBuilder) { + if (!fixedBuffers.empty()) { + auto builder = initFixedBuilder(fixedBuffers.size()); + for (uint64_t i = 0; i < fixedBuffers.size(); ++i) { + builder.init(i, fixedBuffers[i].size()); + for (uint64_t j = 0; j < fixedBuffers[i].size(); ++j) { + builder[i].set(j, fixedBuffers[i][j]); + } + } + } + + if (!varBuffers.empty()) { + auto builder = initVarBuilder(varBuffers.size()); + for (uint64_t i = 0; i < varBuffers.size(); ++i) { + builder.init(i, varBuffers[i].size()); + for (uint64_t j = 0; j < varBuffers[i].size(); ++j) { + builder[i].set(j, varBuffers[i][j]); + } + } + } +} + Status fragment_metadata_to_capnp( const FragmentMetadata& frag_meta, capnp::FragmentMetadata::Builder* frag_meta_builder) { @@ -573,50 +643,46 @@ Status fragment_metadata_to_capnp( } } - auto& tile_min_buffer = frag_meta.loaded_metadata()->tile_min_buffer(); - if (!tile_min_buffer.empty()) { - auto builder = frag_meta_builder->initTileMinBuffer(tile_min_buffer.size()); - for (uint64_t i = 0; i < tile_min_buffer.size(); ++i) { - builder.init(i, tile_min_buffer[i].size()); - for (uint64_t j = 0; j < tile_min_buffer[i].size(); ++j) { - builder[i].set(j, tile_min_buffer[i][j]); - } - } - } - auto& tile_min_var_buffer = - frag_meta.loaded_metadata()->tile_min_var_buffer(); - if (!tile_min_var_buffer.empty()) { - auto builder = - frag_meta_builder->initTileMinVarBuffer(tile_min_var_buffer.size()); - for (uint64_t i = 0; i < tile_min_var_buffer.size(); ++i) { - builder.init(i, tile_min_var_buffer[i].size()); - for (uint64_t j = 0; j < tile_min_var_buffer[i].size(); ++j) { - builder[i].set(j, tile_min_var_buffer[i][j]); - } - } - } - auto& tile_max_buffer = frag_meta.loaded_metadata()->tile_max_buffer(); - if (!tile_max_buffer.empty()) { - auto builder = frag_meta_builder->initTileMaxBuffer(tile_max_buffer.size()); - for (uint64_t i = 0; i < tile_max_buffer.size(); ++i) { - builder.init(i, tile_max_buffer[i].size()); - for (uint64_t j = 0; j < tile_max_buffer[i].size(); ++j) { - builder[i].set(j, tile_max_buffer[i][j]); - } - } - } - auto& tile_max_var_buffer = - frag_meta.loaded_metadata()->tile_max_var_buffer(); - if (!tile_max_var_buffer.empty()) { - auto builder = - frag_meta_builder->initTileMaxVarBuffer(tile_max_var_buffer.size()); - for (uint64_t i = 0; i < tile_max_var_buffer.size(); ++i) { - builder.init(i, tile_max_var_buffer[i].size()); - for (uint64_t j = 0; j < tile_max_var_buffer[i].size(); ++j) { - builder[i].set(j, tile_max_var_buffer[i][j]); - } - } - } + const auto& lm = *frag_meta.loaded_metadata(); + + tile_bounds_to_capnp( + lm.tile_min_buffer(), + lm.tile_min_var_buffer(), + [&frag_meta_builder](auto size) { + return frag_meta_builder->initTileMinBuffer(size); + }, + [&frag_meta_builder](auto size) { + return frag_meta_builder->initTileMinVarBuffer(size); + }); + tile_bounds_to_capnp( + lm.tile_max_buffer(), + lm.tile_max_var_buffer(), + [&frag_meta_builder](auto size) { + return frag_meta_builder->initTileMaxBuffer(size); + }, + [&frag_meta_builder](auto size) { + return frag_meta_builder->initTileMaxVarBuffer(size); + }); + + tile_bounds_to_capnp( + lm.tile_global_order_min_buffer(), + lm.tile_global_order_min_var_buffer(), + [&frag_meta_builder](auto size) { + return frag_meta_builder->initTileGlobalOrderMinBuffer(size); + }, + [&frag_meta_builder](auto size) { + return frag_meta_builder->initTileGlobalOrderMinVarBuffer(size); + }); + tile_bounds_to_capnp( + lm.tile_global_order_max_buffer(), + lm.tile_global_order_min_var_buffer(), + [&frag_meta_builder](auto size) { + return frag_meta_builder->initTileGlobalOrderMaxBuffer(size); + }, + [&frag_meta_builder](auto size) { + return frag_meta_builder->initTileGlobalOrderMaxVarBuffer(size); + }); + auto& tile_sums = frag_meta.loaded_metadata()->tile_sums(); if (!tile_sums.empty()) { auto builder = frag_meta_builder->initTileSums(tile_sums.size()); diff --git a/tiledb/sm/serialization/tiledb-rest.capnp b/tiledb/sm/serialization/tiledb-rest.capnp index a876c0e24a9..78ab7ce4e66 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp +++ b/tiledb/sm/serialization/tiledb-rest.capnp @@ -1093,6 +1093,10 @@ struct FragmentMetadata { # fragment min/max/sum/nullcount offsets processedConditionsOffsets @10 :UInt64; # processed conditions offsets + tileGlobalOrderMinOffsets @11 :List(UInt64); + # global order min tile offsets + tileGlobalOrderMaxOffsets @12 :List(UInt64); + # global order max tile offsets } fileSizes @0 :List(UInt64); @@ -1184,6 +1188,18 @@ struct FragmentMetadata { arraySchemaName @29 :Text; # array schema name + + tileGlobalOrderMinBuffer @30 :List(List(UInt8)); + # tile global order min buffers + + tileGlobalOrderMinVarBuffer @31 :List(List(UInt8)); + # tile global order min buffers for var length dimensions + + tileGlobalOrderMaxBuffer @32 :List(List(UInt8)); + # tile global order max buffers + + tileGlobalOrderMaxVarBuffer @33 :List(List(UInt8)); + # tile global order max buffers for var length dimensions } struct MultiPartUploadState { diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.c++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ index 33d1f2ad1fe..c0829129620 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.c++ +++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ @@ -12,7 +12,7 @@ static const ::capnp::_::AlignedData<209> b_ce5904e6f9410cec = { 127, 216, 135, 181, 36, 146, 125, 181, 10, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 55, 1, 0, 0, 94, 2, 0, 0, + 44, 1, 0, 0, 72, 2, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -232,7 +232,7 @@ static const ::capnp::_::AlignedData<48> b_e3dadf2bf211bc97 = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 98, 2, 0, 0, 148, 2, 0, 0, + 74, 2, 0, 0, 121, 2, 0, 0, 21, 0, 0, 0, 170, 0, 0, 0, 29, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -291,7 +291,7 @@ static const ::capnp::_::AlignedData<38> b_b6c95b4b8111ad36 = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 152, 2, 0, 0, 1, 3, 0, 0, + 123, 2, 0, 0, 224, 2, 0, 0, 21, 0, 0, 0, 202, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -343,7 +343,7 @@ static const ::capnp::_::AlignedData<218> b_a45730f57e0460b4 = { 127, 216, 135, 181, 36, 146, 125, 181, 8, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 5, 3, 0, 0, 23, 6, 0, 0, + 226, 2, 0, 0, 211, 5, 0, 0, 21, 0, 0, 0, 194, 0, 0, 0, 29, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -583,7 +583,7 @@ static const ::capnp::_::AlignedData<50> b_facceeafd4472c68 = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 27, 6, 0, 0, 145, 6, 0, 0, + 213, 5, 0, 0, 69, 6, 0, 0, 21, 0, 0, 0, 226, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -647,7 +647,7 @@ static const ::capnp::_::AlignedData<332> b_d71de32f98e296fe = { 127, 216, 135, 181, 36, 146, 125, 181, 16, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 149, 6, 0, 0, 2, 12, 0, 0, + 71, 6, 0, 0, 124, 11, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -999,7 +999,7 @@ static const ::capnp::_::AlignedData<50> b_d24744ef593dcfbe = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 6, 12, 0, 0, 217, 12, 0, 0, + 126, 11, 0, 0, 74, 12, 0, 0, 21, 0, 0, 0, 42, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1063,7 +1063,7 @@ static const ::capnp::_::AlignedData<175> b_ceff8d62d10cd1de = { 127, 216, 135, 181, 36, 146, 125, 181, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 221, 12, 0, 0, 198, 15, 0, 0, + 76, 12, 0, 0, 22, 15, 0, 0, 21, 0, 0, 0, 10, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1252,7 +1252,7 @@ static const ::capnp::_::AlignedData<161> b_a1b81d67548230d4 = { 127, 216, 135, 181, 36, 146, 125, 181, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 202, 15, 0, 0, 70, 18, 0, 0, + 24, 15, 0, 0, 126, 17, 0, 0, 21, 0, 0, 0, 58, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1429,7 +1429,7 @@ static const ::capnp::_::AlignedData<161> b_92ad78f56de3d76a = { 127, 216, 135, 181, 36, 146, 125, 181, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 74, 18, 0, 0, 247, 20, 0, 0, + 128, 17, 0, 0, 17, 20, 0, 0, 21, 0, 0, 0, 226, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1604,7 +1604,7 @@ static const ::capnp::_::AlignedData<126> b_d00b2f19cc21b9b4 = { 127, 216, 135, 181, 36, 146, 125, 181, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 251, 20, 0, 0, 14, 23, 0, 0, + 19, 20, 0, 0, 16, 22, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1741,7 +1741,7 @@ static const ::capnp::_::AlignedData<144> b_d20a578112fa92a2 = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 18, 23, 0, 0, 88, 26, 0, 0, + 18, 22, 0, 0, 65, 25, 0, 0, 21, 0, 0, 0, 66, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1896,7 +1896,7 @@ static const ::capnp::_::AlignedData<105> b_95e26a84d32d8223 = { 127, 216, 135, 181, 36, 146, 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 92, 26, 0, 0, 231, 28, 0, 0, + 67, 25, 0, 0, 175, 27, 0, 0, 21, 0, 0, 0, 226, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2200,7 +2200,7 @@ static const ::capnp::_::AlignedData<86> b_de030f447664754c = { 127, 216, 135, 181, 36, 146, 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 235, 28, 0, 0, 218, 29, 0, 0, + 177, 27, 0, 0, 147, 28, 0, 0, 21, 0, 0, 0, 202, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2300,7 +2300,7 @@ static const ::capnp::_::AlignedData<48> b_fa787661cd3563a4 = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 222, 29, 0, 0, 27, 30, 0, 0, + 149, 28, 0, 0, 207, 28, 0, 0, 21, 0, 0, 0, 194, 0, 0, 0, 29, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2359,7 +2359,7 @@ static const ::capnp::_::AlignedData<66> b_fda1cb9663a55b70 = { 127, 216, 135, 181, 36, 146, 125, 181, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 31, 30, 0, 0, 129, 30, 0, 0, + 209, 28, 0, 0, 47, 29, 0, 0, 21, 0, 0, 0, 26, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2436,7 +2436,7 @@ static const ::capnp::_::AlignedData<95> b_fd5825d8c6e74d78 = { 127, 216, 135, 181, 36, 146, 125, 181, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 133, 30, 0, 0, 212, 31, 0, 0, + 49, 29, 0, 0, 117, 30, 0, 0, 21, 0, 0, 0, 234, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2542,7 +2542,7 @@ static const ::capnp::_::AlignedData<75> b_e7175047415b3f97 = { 127, 216, 135, 181, 36, 146, 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 216, 31, 0, 0, 152, 33, 0, 0, + 119, 30, 0, 0, 32, 32, 0, 0, 21, 0, 0, 0, 202, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2845,7 +2845,7 @@ static const ::capnp::_::AlignedData<39> b_bc4583f733eac4f5 = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 156, 33, 0, 0, 211, 33, 0, 0, + 34, 32, 0, 0, 87, 32, 0, 0, 21, 0, 0, 0, 10, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2898,7 +2898,7 @@ static const ::capnp::_::AlignedData<50> b_f179c194ae71718c = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 215, 33, 0, 0, 77, 34, 0, 0, + 89, 32, 0, 0, 201, 32, 0, 0, 21, 0, 0, 0, 178, 0, 0, 0, 29, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2965,7 +2965,7 @@ static const ::capnp::_::AlignedData<49> b_db5514c8aaf6faea = { 140, 113, 113, 174, 148, 193, 121, 241, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 15, 34, 0, 0, 74, 34, 0, 0, + 143, 32, 0, 0, 199, 32, 0, 0, 21, 0, 0, 0, 226, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -3025,7 +3025,7 @@ static const ::capnp::_::AlignedData<42> b_a9d01efb4b5d8599 = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 81, 34, 0, 0, 211, 34, 0, 0, + 203, 32, 0, 0, 71, 33, 0, 0, 21, 0, 0, 0, 10, 1, 0, 0, 37, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -3081,7 +3081,7 @@ static const ::capnp::_::AlignedData<54> b_ff915a194348206a = { 153, 133, 93, 75, 251, 30, 208, 169, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 136, 34, 0, 0, 208, 34, 0, 0, + 0, 33, 0, 0, 69, 33, 0, 0, 21, 0, 0, 0, 58, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -3149,7 +3149,7 @@ static const ::capnp::_::AlignedData<41> b_c6b5bb09d4611252 = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 215, 34, 0, 0, 73, 35, 0, 0, + 73, 33, 0, 0, 181, 33, 0, 0, 21, 0, 0, 0, 226, 0, 0, 0, 33, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -3204,7 +3204,7 @@ static const ::capnp::_::AlignedData<50> b_884e0a5f2521a5c6 = { 82, 18, 97, 212, 9, 187, 181, 198, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 9, 35, 0, 0, 70, 35, 0, 0, + 121, 33, 0, 0, 179, 33, 0, 0, 21, 0, 0, 0, 18, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -3265,7 +3265,7 @@ static const ::capnp::_::AlignedData<41> b_a83707d3ba24dd32 = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 77, 35, 0, 0, 189, 35, 0, 0, + 183, 33, 0, 0, 33, 34, 0, 0, 21, 0, 0, 0, 218, 0, 0, 0, 33, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -3320,7 +3320,7 @@ static const ::capnp::_::AlignedData<50> b_a9caccb4333a2baf = { 50, 221, 36, 186, 211, 7, 55, 168, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 126, 35, 0, 0, 186, 35, 0, 0, + 230, 33, 0, 0, 31, 34, 0, 0, 21, 0, 0, 0, 10, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -3381,7 +3381,7 @@ static const ::capnp::_::AlignedData<41> b_d3c5983c670e0f42 = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 193, 35, 0, 0, 51, 36, 0, 0, + 35, 34, 0, 0, 143, 34, 0, 0, 21, 0, 0, 0, 226, 0, 0, 0, 33, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -3436,7 +3436,7 @@ static const ::capnp::_::AlignedData<50> b_82d7452da7cd249a = { 66, 15, 14, 103, 60, 152, 197, 211, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 243, 35, 0, 0, 48, 36, 0, 0, + 83, 34, 0, 0, 141, 34, 0, 0, 21, 0, 0, 0, 18, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -3497,7 +3497,7 @@ static const ::capnp::_::AlignedData<41> b_9354174d952207d2 = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 55, 36, 0, 0, 171, 36, 0, 0, + 145, 34, 0, 0, 255, 34, 0, 0, 21, 0, 0, 0, 234, 0, 0, 0, 33, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -3552,7 +3552,7 @@ static const ::capnp::_::AlignedData<50> b_bb80cdbc3f312934 = { 210, 7, 34, 149, 77, 23, 84, 147, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 106, 36, 0, 0, 168, 36, 0, 0, + 194, 34, 0, 0, 253, 34, 0, 0, 21, 0, 0, 0, 26, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -3613,7 +3613,7 @@ static const ::capnp::_::AlignedData<49> b_c7e036a11506a1a4 = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 175, 36, 0, 0, 34, 37, 0, 0, + 1, 35, 0, 0, 108, 35, 0, 0, 21, 0, 0, 0, 194, 0, 0, 0, 29, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -3677,7 +3677,7 @@ static const ::capnp::_::AlignedData<91> b_8bf6c1d37e748294 = { 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 38, 37, 0, 0, 157, 38, 0, 0, + 110, 35, 0, 0, 217, 36, 0, 0, 21, 0, 0, 0, 58, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -3782,7 +3782,7 @@ static const ::capnp::_::AlignedData<148> b_8ba60147a0e6735e = { 127, 216, 135, 181, 36, 146, 125, 181, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 161, 38, 0, 0, 189, 40, 0, 0, + 219, 36, 0, 0, 225, 38, 0, 0, 21, 0, 0, 0, 202, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -3948,7 +3948,7 @@ static const ::capnp::_::AlignedData<107> b_86cfc12d74ed4aa0 = { 127, 216, 135, 181, 36, 146, 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 193, 40, 0, 0, 66, 42, 0, 0, + 227, 38, 0, 0, 83, 40, 0, 0, 21, 0, 0, 0, 10, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -4066,7 +4066,7 @@ static const ::capnp::_::AlignedData<66> b_f7aa276767b422e7 = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 70, 42, 0, 0, 96, 43, 0, 0, + 85, 40, 0, 0, 100, 41, 0, 0, 21, 0, 0, 0, 50, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -4146,7 +4146,7 @@ static const ::capnp::_::AlignedData<159> b_dba20dec138adac9 = { 127, 216, 135, 181, 36, 146, 125, 181, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 100, 43, 0, 0, 155, 45, 0, 0, + 102, 41, 0, 0, 134, 43, 0, 0, 21, 0, 0, 0, 218, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -4325,7 +4325,7 @@ static const ::capnp::_::AlignedData<156> b_ff14003c70494585 = { 127, 216, 135, 181, 36, 146, 125, 181, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 159, 45, 0, 0, 169, 49, 0, 0, + 136, 43, 0, 0, 106, 47, 0, 0, 21, 0, 0, 0, 50, 1, 0, 0, 37, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -4499,7 +4499,7 @@ static const ::capnp::_::AlignedData<84> b_f86b7bf97823250f = { 133, 69, 73, 112, 60, 0, 20, 255, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 223, 45, 0, 0, 97, 46, 0, 0, + 197, 43, 0, 0, 66, 44, 0, 0, 21, 0, 0, 0, 162, 1, 0, 0, 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -4597,7 +4597,7 @@ static const ::capnp::_::AlignedData<91> b_fdd9e47288724221 = { 133, 69, 73, 112, 60, 0, 20, 255, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 103, 46, 0, 0, 239, 46, 0, 0, + 70, 44, 0, 0, 201, 44, 0, 0, 21, 0, 0, 0, 98, 1, 0, 0, 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -4702,7 +4702,7 @@ static const ::capnp::_::AlignedData<84> b_cbe1e7c13508aa2c = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 173, 49, 0, 0, 24, 51, 0, 0, + 108, 47, 0, 0, 203, 48, 0, 0, 21, 0, 0, 0, 226, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -4800,7 +4800,7 @@ static const ::capnp::_::AlignedData<82> b_dac6a7f675c57409 = { 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 28, 51, 0, 0, 98, 52, 0, 0, + 205, 48, 0, 0, 5, 50, 0, 0, 21, 0, 0, 0, 18, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -4893,7 +4893,7 @@ static const ::capnp::_::AlignedData<148> b_afc739d5c01e6496 = { 127, 216, 135, 181, 36, 146, 125, 181, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 102, 52, 0, 0, 20, 55, 0, 0, + 7, 50, 0, 0, 154, 52, 0, 0, 21, 0, 0, 0, 210, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -5055,7 +5055,7 @@ static const ::capnp::_::AlignedData<74> b_eaf57cb9871fc06f = { 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 24, 55, 0, 0, 43, 56, 0, 0, + 156, 52, 0, 0, 164, 53, 0, 0, 21, 0, 0, 0, 226, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -5144,7 +5144,7 @@ static const ::capnp::_::AlignedData<114> b_e19754f813ccf79c = { 127, 216, 135, 181, 36, 146, 125, 181, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 47, 56, 0, 0, 29, 58, 0, 0, + 166, 53, 0, 0, 128, 55, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -5275,7 +5275,7 @@ static const ::capnp::_::AlignedData<50> b_d74f5fed155d316c = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 58, 0, 0, 166, 58, 0, 0, + 130, 55, 0, 0, 255, 55, 0, 0, 21, 0, 0, 0, 202, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -5340,7 +5340,7 @@ static const ::capnp::_::AlignedData<80> b_def87cead82188e7 = { 127, 216, 135, 181, 36, 146, 125, 181, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 170, 58, 0, 0, 166, 59, 0, 0, + 1, 56, 0, 0, 239, 56, 0, 0, 21, 0, 0, 0, 10, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -5431,7 +5431,7 @@ static const ::capnp::_::AlignedData<49> b_c1a2d010de779de5 = { 127, 216, 135, 181, 36, 146, 125, 181, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 170, 59, 0, 0, 63, 60, 0, 0, + 241, 56, 0, 0, 126, 57, 0, 0, 21, 0, 0, 0, 2, 1, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -5491,7 +5491,7 @@ static const ::capnp::_::AlignedData<77> b_c86c77b5f6a2bf0f = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 67, 60, 0, 0, 69, 61, 0, 0, + 128, 57, 0, 0, 121, 58, 0, 0, 21, 0, 0, 0, 10, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -5583,7 +5583,7 @@ static const ::capnp::_::AlignedData<97> b_9b9a5fc7713a8692 = { 127, 216, 135, 181, 36, 146, 125, 181, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 73, 61, 0, 0, 170, 62, 0, 0, + 123, 58, 0, 0, 203, 59, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -5697,7 +5697,7 @@ static const ::capnp::_::AlignedData<388> b_96ba49d0f8b23ccc = { 127, 216, 135, 181, 36, 146, 125, 181, 17, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 174, 62, 0, 0, 115, 71, 0, 0, + 205, 59, 0, 0, 78, 68, 0, 0, 21, 0, 0, 0, 194, 0, 0, 0, 29, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6108,7 +6108,7 @@ static const ::capnp::_::AlignedData<70> b_9df6f2a42c4e5f0b = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 119, 71, 0, 0, 146, 72, 0, 0, + 80, 68, 0, 0, 96, 69, 0, 0, 21, 0, 0, 0, 10, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6192,7 +6192,7 @@ static const ::capnp::_::AlignedData<40> b_a18264549448ece3 = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 150, 72, 0, 0, 50, 73, 0, 0, + 98, 69, 0, 0, 251, 69, 0, 0, 21, 0, 0, 0, 42, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6246,7 +6246,7 @@ static const ::capnp::_::AlignedData<84> b_9be1921b07e6cd2d = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 54, 73, 0, 0, 129, 74, 0, 0, + 253, 69, 0, 0, 58, 71, 0, 0, 21, 0, 0, 0, 50, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6341,7 +6341,7 @@ static const ::capnp::_::AlignedData<42> b_9737dcafdfce31bb = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 133, 74, 0, 0, 147, 75, 0, 0, + 60, 71, 0, 0, 61, 72, 0, 0, 21, 0, 0, 0, 2, 1, 0, 0, 33, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6397,7 +6397,7 @@ static const ::capnp::_::AlignedData<97> b_926fe1c3b12ed651 = { 187, 49, 206, 223, 175, 220, 55, 151, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 201, 74, 0, 0, 77, 75, 0, 0, + 125, 71, 0, 0, 251, 71, 0, 0, 21, 0, 0, 0, 114, 1, 0, 0, 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6505,7 +6505,7 @@ static const ::capnp::_::AlignedData<311> b_9317f20ce509d918 = { 127, 216, 135, 181, 36, 146, 125, 181, 12, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 151, 75, 0, 0, 240, 81, 0, 0, + 63, 72, 0, 0, 94, 78, 0, 0, 21, 0, 0, 0, 10, 1, 0, 0, 37, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6831,7 +6831,7 @@ static const ::capnp::_::AlignedData<68> b_d9d27c082dec9e26 = { 24, 217, 9, 229, 12, 242, 23, 147, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 224, 75, 0, 0, 77, 76, 0, 0, + 133, 72, 0, 0, 238, 72, 0, 0, 21, 0, 0, 0, 130, 1, 0, 0, 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6910,7 +6910,7 @@ static const ::capnp::_::AlignedData<69> b_9eb745af522d087c = { 24, 217, 9, 229, 12, 242, 23, 147, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 83, 76, 0, 0, 198, 76, 0, 0, + 242, 72, 0, 0, 97, 73, 0, 0, 21, 0, 0, 0, 234, 1, 0, 0, 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6990,7 +6990,7 @@ static const ::capnp::_::AlignedData<96> b_8cd4e323f1feea3b = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 246, 81, 0, 0, 149, 83, 0, 0, + 97, 78, 0, 0, 238, 79, 0, 0, 21, 0, 0, 0, 50, 1, 0, 0, 37, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7104,7 +7104,7 @@ static const ::capnp::_::AlignedData<69> b_92c8467685565269 = { 59, 234, 254, 241, 35, 227, 212, 140, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 57, 82, 0, 0, 185, 82, 0, 0, + 162, 78, 0, 0, 29, 79, 0, 0, 21, 0, 0, 0, 138, 1, 0, 0, 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7184,7 +7184,7 @@ static const ::capnp::_::AlignedData<69> b_bda7916926591c22 = { 59, 234, 254, 241, 35, 227, 212, 140, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 191, 82, 0, 0, 60, 83, 0, 0, + 33, 79, 0, 0, 153, 79, 0, 0, 21, 0, 0, 0, 138, 1, 0, 0, 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7264,7 +7264,7 @@ static const ::capnp::_::AlignedData<35> b_c95970eb9310dc9c = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 153, 83, 0, 0, 218, 83, 0, 0, + 240, 79, 0, 0, 46, 80, 0, 0, 21, 0, 0, 0, 50, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7313,7 +7313,7 @@ static const ::capnp::_::AlignedData<67> b_d42e7b38b33e3d29 = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 222, 83, 0, 0, 181, 84, 0, 0, + 48, 80, 0, 0, 254, 80, 0, 0, 21, 0, 0, 0, 42, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7394,7 +7394,7 @@ static const ::capnp::_::AlignedData<110> b_a000530ab1d17816 = { 127, 216, 135, 181, 36, 146, 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 185, 84, 0, 0, 4, 86, 0, 0, + 0, 81, 0, 0, 63, 82, 0, 0, 21, 0, 0, 0, 250, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7523,7 +7523,7 @@ static const ::capnp::_::AlignedData<50> b_838188de0fd57580 = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 8, 86, 0, 0, 133, 86, 0, 0, + 65, 82, 0, 0, 184, 82, 0, 0, 21, 0, 0, 0, 2, 1, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7588,7 +7588,7 @@ static const ::capnp::_::AlignedData<80> b_c41bcc7e8cc58f18 = { 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 137, 86, 0, 0, 113, 87, 0, 0, + 186, 82, 0, 0, 150, 83, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7679,7 +7679,7 @@ static const ::capnp::_::AlignedData<61> b_dcdd20e1b79e915a = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 87, 0, 0, 145, 88, 0, 0, + 152, 83, 0, 0, 165, 84, 0, 0, 21, 0, 0, 0, 194, 0, 0, 0, 29, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7755,7 +7755,7 @@ static const ::capnp::_::AlignedData<55> b_a2ea10c715b475c1 = { 90, 145, 158, 183, 225, 32, 221, 220, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 148, 87, 0, 0, 56, 88, 0, 0, + 180, 83, 0, 0, 82, 84, 0, 0, 21, 0, 0, 0, 42, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7825,7 +7825,7 @@ static const ::capnp::_::AlignedData<65> b_c4e54a63294eddb7 = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 149, 88, 0, 0, 15, 90, 0, 0, + 167, 84, 0, 0, 19, 86, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7905,7 +7905,7 @@ static const ::capnp::_::AlignedData<83> b_83b01e46759bde40 = { 183, 221, 78, 41, 99, 74, 229, 196, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 173, 88, 0, 0, 137, 89, 0, 0, + 190, 84, 0, 0, 148, 85, 0, 0, 21, 0, 0, 0, 138, 1, 0, 0, 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -8002,7 +8002,7 @@ static const ::capnp::_::AlignedData<65> b_fb7f36ad4d8ffe84 = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 19, 90, 0, 0, 42, 91, 0, 0, + 21, 86, 0, 0, 30, 87, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -8082,7 +8082,7 @@ static const ::capnp::_::AlignedData<37> b_d5fd459ad75e86a9 = { 132, 254, 143, 77, 173, 54, 127, 251, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 71, 90, 0, 0, 189, 90, 0, 0, + 70, 86, 0, 0, 183, 86, 0, 0, 21, 0, 0, 0, 138, 1, 0, 0, 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -8130,7 +8130,7 @@ static const ::capnp::_::AlignedData<120> b_c0c730b5390f4427 = { 127, 216, 135, 181, 36, 146, 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 46, 91, 0, 0, 72, 94, 0, 0, + 32, 87, 0, 0, 38, 90, 0, 0, 21, 0, 0, 0, 26, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -8270,7 +8270,7 @@ static const ::capnp::_::AlignedData<81> b_d8bd3c0dec37b773 = { 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 76, 94, 0, 0, 56, 95, 0, 0, + 40, 90, 0, 0, 11, 91, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -8355,18 +8355,18 @@ const ::capnp::_::RawSchema s_d8bd3c0dec37b773 = { 0, 3, i_d8bd3c0dec37b773, nullptr, nullptr, { &s_d8bd3c0dec37b773, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<629> b_cde352fc27e7ca61 = { +static const ::capnp::_::AlignedData<733> b_cde352fc27e7ca61 = { { 0, 0, 0, 0, 6, 0, 6, 0, 97, 202, 231, 39, 252, 82, 227, 205, 18, 0, 0, 0, 1, 0, 4, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 23, 0, 7, 0, 0, 0, 0, 0, + 27, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 60, 95, 0, 0, 228, 107, 0, 0, + 13, 91, 0, 0, 134, 105, 0, 0, 21, 0, 0, 0, 26, 1, 0, 0, 37, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 0, 0, 0, 151, 6, 0, 0, + 53, 0, 0, 0, 119, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -8380,217 +8380,245 @@ static const ::capnp::_::AlignedData<629> b_cde352fc27e7ca61 = { 71, 101, 110, 101, 114, 105, 99, 84, 105, 108, 101, 79, 102, 102, 115, 101, 116, 115, 0, 0, 0, 0, 0, 0, - 120, 0, 0, 0, 3, 0, 4, 0, + 136, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 57, 3, 0, 0, 82, 0, 0, 0, + 169, 3, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 56, 3, 0, 0, 3, 0, 1, 0, - 84, 3, 0, 0, 2, 0, 1, 0, + 168, 3, 0, 0, 3, 0, 1, 0, + 196, 3, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 81, 3, 0, 0, 106, 0, 0, 0, + 193, 3, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 80, 3, 0, 0, 3, 0, 1, 0, - 108, 3, 0, 0, 2, 0, 1, 0, + 192, 3, 0, 0, 3, 0, 1, 0, + 220, 3, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 105, 3, 0, 0, 146, 0, 0, 0, + 217, 3, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 108, 3, 0, 0, 3, 0, 1, 0, - 136, 3, 0, 0, 2, 0, 1, 0, + 220, 3, 0, 0, 3, 0, 1, 0, + 248, 3, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 133, 3, 0, 0, 98, 0, 0, 0, + 245, 3, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 132, 3, 0, 0, 3, 0, 1, 0, - 144, 3, 0, 0, 2, 0, 1, 0, + 244, 3, 0, 0, 3, 0, 1, 0, + 0, 4, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 141, 3, 0, 0, 114, 0, 0, 0, + 253, 3, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 140, 3, 0, 0, 3, 0, 1, 0, - 152, 3, 0, 0, 2, 0, 1, 0, + 252, 3, 0, 0, 3, 0, 1, 0, + 8, 4, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 149, 3, 0, 0, 114, 0, 0, 0, + 5, 4, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 148, 3, 0, 0, 3, 0, 1, 0, - 160, 3, 0, 0, 2, 0, 1, 0, + 4, 4, 0, 0, 3, 0, 1, 0, + 16, 4, 0, 0, 2, 0, 1, 0, 6, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 157, 3, 0, 0, 114, 0, 0, 0, + 13, 4, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 156, 3, 0, 0, 3, 0, 1, 0, - 168, 3, 0, 0, 2, 0, 1, 0, + 12, 4, 0, 0, 3, 0, 1, 0, + 24, 4, 0, 0, 2, 0, 1, 0, 7, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 165, 3, 0, 0, 114, 0, 0, 0, + 21, 4, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 164, 3, 0, 0, 3, 0, 1, 0, - 176, 3, 0, 0, 2, 0, 1, 0, + 20, 4, 0, 0, 3, 0, 1, 0, + 32, 4, 0, 0, 2, 0, 1, 0, 8, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 173, 3, 0, 0, 98, 0, 0, 0, + 29, 4, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 172, 3, 0, 0, 3, 0, 1, 0, - 216, 3, 0, 0, 2, 0, 1, 0, + 28, 4, 0, 0, 3, 0, 1, 0, + 72, 4, 0, 0, 2, 0, 1, 0, 9, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 213, 3, 0, 0, 122, 0, 0, 0, + 69, 4, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 212, 3, 0, 0, 3, 0, 1, 0, - 0, 4, 0, 0, 2, 0, 1, 0, + 68, 4, 0, 0, 3, 0, 1, 0, + 112, 4, 0, 0, 2, 0, 1, 0, 10, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 253, 3, 0, 0, 106, 0, 0, 0, + 109, 4, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 252, 3, 0, 0, 3, 0, 1, 0, - 40, 4, 0, 0, 2, 0, 1, 0, + 108, 4, 0, 0, 3, 0, 1, 0, + 152, 4, 0, 0, 2, 0, 1, 0, 11, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 4, 0, 0, 162, 0, 0, 0, + 149, 4, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 40, 4, 0, 0, 3, 0, 1, 0, - 84, 4, 0, 0, 2, 0, 1, 0, + 152, 4, 0, 0, 3, 0, 1, 0, + 196, 4, 0, 0, 2, 0, 1, 0, 12, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 81, 4, 0, 0, 114, 0, 0, 0, + 193, 4, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 80, 4, 0, 0, 3, 0, 1, 0, - 124, 4, 0, 0, 2, 0, 1, 0, + 192, 4, 0, 0, 3, 0, 1, 0, + 236, 4, 0, 0, 2, 0, 1, 0, 13, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 121, 4, 0, 0, 138, 0, 0, 0, + 233, 4, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 124, 4, 0, 0, 3, 0, 1, 0, - 168, 4, 0, 0, 2, 0, 1, 0, + 236, 4, 0, 0, 3, 0, 1, 0, + 24, 5, 0, 0, 2, 0, 1, 0, 14, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 165, 4, 0, 0, 114, 0, 0, 0, + 21, 5, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 164, 4, 0, 0, 3, 0, 1, 0, - 208, 4, 0, 0, 2, 0, 1, 0, + 20, 5, 0, 0, 3, 0, 1, 0, + 64, 5, 0, 0, 2, 0, 1, 0, 15, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 205, 4, 0, 0, 138, 0, 0, 0, + 61, 5, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 208, 4, 0, 0, 3, 0, 1, 0, - 252, 4, 0, 0, 2, 0, 1, 0, + 64, 5, 0, 0, 3, 0, 1, 0, + 108, 5, 0, 0, 2, 0, 1, 0, 16, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 249, 4, 0, 0, 74, 0, 0, 0, + 105, 5, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 248, 4, 0, 0, 3, 0, 1, 0, - 36, 5, 0, 0, 2, 0, 1, 0, + 104, 5, 0, 0, 3, 0, 1, 0, + 148, 5, 0, 0, 2, 0, 1, 0, 17, 0, 0, 0, 13, 0, 0, 0, 0, 0, 1, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 5, 0, 0, 122, 0, 0, 0, + 145, 5, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 32, 5, 0, 0, 3, 0, 1, 0, - 76, 5, 0, 0, 2, 0, 1, 0, + 144, 5, 0, 0, 3, 0, 1, 0, + 188, 5, 0, 0, 2, 0, 1, 0, 18, 0, 0, 0, 14, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 73, 5, 0, 0, 106, 0, 0, 0, + 185, 5, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 72, 5, 0, 0, 3, 0, 1, 0, - 116, 5, 0, 0, 2, 0, 1, 0, + 184, 5, 0, 0, 3, 0, 1, 0, + 228, 5, 0, 0, 2, 0, 1, 0, 19, 0, 0, 0, 15, 0, 0, 0, 0, 0, 1, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 113, 5, 0, 0, 106, 0, 0, 0, + 225, 5, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 112, 5, 0, 0, 3, 0, 1, 0, - 156, 5, 0, 0, 2, 0, 1, 0, + 224, 5, 0, 0, 3, 0, 1, 0, + 12, 6, 0, 0, 2, 0, 1, 0, 20, 0, 0, 0, 16, 0, 0, 0, 0, 0, 1, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 153, 5, 0, 0, 106, 0, 0, 0, + 9, 6, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 152, 5, 0, 0, 3, 0, 1, 0, - 180, 5, 0, 0, 2, 0, 1, 0, + 8, 6, 0, 0, 3, 0, 1, 0, + 36, 6, 0, 0, 2, 0, 1, 0, 21, 0, 0, 0, 17, 0, 0, 0, 0, 0, 1, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 177, 5, 0, 0, 154, 0, 0, 0, + 33, 6, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 180, 5, 0, 0, 3, 0, 1, 0, - 208, 5, 0, 0, 2, 0, 1, 0, + 36, 6, 0, 0, 3, 0, 1, 0, + 64, 6, 0, 0, 2, 0, 1, 0, 22, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 205, 5, 0, 0, 66, 0, 0, 0, + 61, 6, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 200, 5, 0, 0, 3, 0, 1, 0, - 212, 5, 0, 0, 2, 0, 1, 0, + 56, 6, 0, 0, 3, 0, 1, 0, + 68, 6, 0, 0, 2, 0, 1, 0, 23, 0, 0, 0, 18, 0, 0, 0, 0, 0, 1, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 209, 5, 0, 0, 122, 0, 0, 0, + 65, 6, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 208, 5, 0, 0, 3, 0, 1, 0, - 236, 5, 0, 0, 2, 0, 1, 0, + 64, 6, 0, 0, 3, 0, 1, 0, + 92, 6, 0, 0, 2, 0, 1, 0, 24, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 233, 5, 0, 0, 130, 0, 0, 0, + 89, 6, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 232, 5, 0, 0, 3, 0, 1, 0, - 244, 5, 0, 0, 2, 0, 1, 0, + 88, 6, 0, 0, 3, 0, 1, 0, + 100, 6, 0, 0, 2, 0, 1, 0, 25, 0, 0, 0, 19, 0, 0, 0, 0, 0, 1, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 241, 5, 0, 0, 122, 0, 0, 0, + 97, 6, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 240, 5, 0, 0, 3, 0, 1, 0, - 252, 5, 0, 0, 2, 0, 1, 0, + 96, 6, 0, 0, 3, 0, 1, 0, + 108, 6, 0, 0, 2, 0, 1, 0, 26, 0, 0, 0, 20, 0, 0, 0, 0, 0, 1, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 249, 5, 0, 0, 50, 0, 0, 0, + 105, 6, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 244, 5, 0, 0, 3, 0, 1, 0, - 0, 6, 0, 0, 2, 0, 1, 0, + 100, 6, 0, 0, 3, 0, 1, 0, + 112, 6, 0, 0, 2, 0, 1, 0, 27, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 253, 5, 0, 0, 178, 0, 0, 0, + 109, 6, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 6, 0, 0, 3, 0, 1, 0, - 12, 6, 0, 0, 2, 0, 1, 0, + 112, 6, 0, 0, 3, 0, 1, 0, + 124, 6, 0, 0, 2, 0, 1, 0, 28, 0, 0, 0, 21, 0, 0, 0, 0, 0, 1, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 9, 6, 0, 0, 82, 0, 0, 0, + 121, 6, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 8, 6, 0, 0, 3, 0, 1, 0, - 20, 6, 0, 0, 2, 0, 1, 0, + 120, 6, 0, 0, 3, 0, 1, 0, + 132, 6, 0, 0, 2, 0, 1, 0, 29, 0, 0, 0, 22, 0, 0, 0, 0, 0, 1, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 17, 6, 0, 0, 130, 0, 0, 0, + 129, 6, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 128, 6, 0, 0, 3, 0, 1, 0, + 140, 6, 0, 0, 2, 0, 1, 0, + 30, 0, 0, 0, 23, 0, 0, 0, + 0, 0, 1, 0, 30, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 137, 6, 0, 0, 202, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 144, 6, 0, 0, 3, 0, 1, 0, + 188, 6, 0, 0, 2, 0, 1, 0, + 31, 0, 0, 0, 24, 0, 0, 0, + 0, 0, 1, 0, 31, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 185, 6, 0, 0, 226, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 192, 6, 0, 0, 3, 0, 1, 0, + 236, 6, 0, 0, 2, 0, 1, 0, + 32, 0, 0, 0, 25, 0, 0, 0, + 0, 0, 1, 0, 32, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 233, 6, 0, 0, 202, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 240, 6, 0, 0, 3, 0, 1, 0, + 28, 7, 0, 0, 2, 0, 1, 0, + 33, 0, 0, 0, 26, 0, 0, 0, + 0, 0, 1, 0, 33, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 25, 7, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 16, 6, 0, 0, 3, 0, 1, 0, - 28, 6, 0, 0, 2, 0, 1, 0, + 32, 7, 0, 0, 3, 0, 1, 0, + 76, 7, 0, 0, 2, 0, 1, 0, 102, 105, 108, 101, 83, 105, 122, 101, 115, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, @@ -8983,6 +9011,82 @@ static const ::capnp::_::AlignedData<629> b_cde352fc27e7ca61 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 71, 108, 111, 98, + 97, 108, 79, 114, 100, 101, 114, 77, + 105, 110, 66, 117, 102, 102, 101, 114, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 6, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 71, 108, 111, 98, + 97, 108, 79, 114, 100, 101, 114, 77, + 105, 110, 86, 97, 114, 66, 117, 102, + 102, 101, 114, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 6, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 71, 108, 111, 98, + 97, 108, 79, 114, 100, 101, 114, 77, + 97, 120, 66, 117, 102, 102, 101, 114, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 6, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 71, 108, 111, 98, + 97, 108, 79, 114, 100, 101, 114, 77, + 97, 120, 86, 97, 114, 66, 117, 102, + 102, 101, 114, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 6, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } }; @@ -8992,25 +9096,25 @@ static const ::capnp::_::RawSchema* const d_cde352fc27e7ca61[] = { &s_89aa8f4e88036b9e, &s_a18264549448ece3, }; -static const uint16_t m_cde352fc27e7ca61[] = {29, 0, 2, 1, 19, 18, 21, 20, 3, 28, 27, 5, 4, 24, 25, 26, 6, 7, 14, 15, 12, 13, 17, 8, 16, 11, 9, 10, 23, 22}; -static const uint16_t i_cde352fc27e7ca61[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29}; +static const uint16_t m_cde352fc27e7ca61[] = {29, 0, 2, 1, 19, 18, 21, 20, 3, 28, 27, 5, 4, 24, 25, 26, 6, 32, 33, 30, 31, 7, 14, 15, 12, 13, 17, 8, 16, 11, 9, 10, 23, 22}; +static const uint16_t i_cde352fc27e7ca61[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33}; const ::capnp::_::RawSchema s_cde352fc27e7ca61 = { - 0xcde352fc27e7ca61, b_cde352fc27e7ca61.words, 629, d_cde352fc27e7ca61, m_cde352fc27e7ca61, - 2, 30, i_cde352fc27e7ca61, nullptr, nullptr, { &s_cde352fc27e7ca61, nullptr, nullptr, 0, 0, nullptr }, false + 0xcde352fc27e7ca61, b_cde352fc27e7ca61.words, 733, d_cde352fc27e7ca61, m_cde352fc27e7ca61, + 2, 34, i_cde352fc27e7ca61, nullptr, nullptr, { &s_cde352fc27e7ca61, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<236> b_89aa8f4e88036b9e = { +static const ::capnp::_::AlignedData<280> b_89aa8f4e88036b9e = { { 0, 0, 0, 0, 6, 0, 6, 0, 158, 107, 3, 136, 78, 143, 170, 137, 35, 0, 0, 0, 1, 0, 3, 0, 97, 202, 231, 39, 252, 82, 227, 205, - 8, 0, 7, 0, 0, 0, 0, 0, + 10, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 89, 95, 0, 0, 173, 98, 0, 0, + 41, 91, 0, 0, 24, 95, 0, 0, 21, 0, 0, 0, 178, 1, 0, 0, 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 111, 2, 0, 0, + 41, 0, 0, 0, 223, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -9021,84 +9125,98 @@ static const ::capnp::_::AlignedData<236> b_89aa8f4e88036b9e = { 105, 99, 84, 105, 108, 101, 79, 102, 102, 115, 101, 116, 115, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 44, 0, 0, 0, 3, 0, 4, 0, + 52, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 1, 0, 0, 50, 0, 0, 0, + 93, 1, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 32, 1, 0, 0, 3, 0, 1, 0, - 44, 1, 0, 0, 2, 0, 1, 0, + 88, 1, 0, 0, 3, 0, 1, 0, + 100, 1, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 1, 0, 0, 98, 0, 0, 0, + 97, 1, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 40, 1, 0, 0, 3, 0, 1, 0, - 68, 1, 0, 0, 2, 0, 1, 0, + 96, 1, 0, 0, 3, 0, 1, 0, + 124, 1, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 65, 1, 0, 0, 122, 0, 0, 0, + 121, 1, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 64, 1, 0, 0, 3, 0, 1, 0, - 92, 1, 0, 0, 2, 0, 1, 0, + 120, 1, 0, 0, 3, 0, 1, 0, + 148, 1, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 89, 1, 0, 0, 106, 0, 0, 0, + 145, 1, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 88, 1, 0, 0, 3, 0, 1, 0, - 116, 1, 0, 0, 2, 0, 1, 0, + 144, 1, 0, 0, 3, 0, 1, 0, + 172, 1, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 113, 1, 0, 0, 162, 0, 0, 0, + 169, 1, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 1, 0, 0, 3, 0, 1, 0, - 144, 1, 0, 0, 2, 0, 1, 0, + 172, 1, 0, 0, 3, 0, 1, 0, + 200, 1, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 141, 1, 0, 0, 122, 0, 0, 0, + 197, 1, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 140, 1, 0, 0, 3, 0, 1, 0, - 168, 1, 0, 0, 2, 0, 1, 0, + 196, 1, 0, 0, 3, 0, 1, 0, + 224, 1, 0, 0, 2, 0, 1, 0, 6, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 165, 1, 0, 0, 122, 0, 0, 0, + 221, 1, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 164, 1, 0, 0, 3, 0, 1, 0, - 192, 1, 0, 0, 2, 0, 1, 0, + 220, 1, 0, 0, 3, 0, 1, 0, + 248, 1, 0, 0, 2, 0, 1, 0, 7, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 189, 1, 0, 0, 122, 0, 0, 0, + 245, 1, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 188, 1, 0, 0, 3, 0, 1, 0, - 216, 1, 0, 0, 2, 0, 1, 0, + 244, 1, 0, 0, 3, 0, 1, 0, + 16, 2, 0, 0, 2, 0, 1, 0, 8, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 213, 1, 0, 0, 170, 0, 0, 0, + 13, 2, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 216, 1, 0, 0, 3, 0, 1, 0, - 244, 1, 0, 0, 2, 0, 1, 0, + 16, 2, 0, 0, 3, 0, 1, 0, + 44, 2, 0, 0, 2, 0, 1, 0, 9, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 241, 1, 0, 0, 10, 1, 0, 0, + 41, 2, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 252, 1, 0, 0, 3, 0, 1, 0, - 8, 2, 0, 0, 2, 0, 1, 0, + 52, 2, 0, 0, 3, 0, 1, 0, + 64, 2, 0, 0, 2, 0, 1, 0, 10, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 5, 2, 0, 0, 218, 0, 0, 0, + 61, 2, 0, 0, 218, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 68, 2, 0, 0, 3, 0, 1, 0, + 80, 2, 0, 0, 2, 0, 1, 0, + 11, 0, 0, 0, 8, 0, 0, 0, + 0, 0, 1, 0, 11, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 77, 2, 0, 0, 210, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 84, 2, 0, 0, 3, 0, 1, 0, + 112, 2, 0, 0, 2, 0, 1, 0, + 12, 0, 0, 0, 9, 0, 0, 0, + 0, 0, 1, 0, 12, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 2, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 12, 2, 0, 0, 3, 0, 1, 0, - 24, 2, 0, 0, 2, 0, 1, 0, + 116, 2, 0, 0, 3, 0, 1, 0, + 144, 2, 0, 0, 2, 0, 1, 0, 114, 116, 114, 101, 101, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -9234,16 +9352,46 @@ static const ::capnp::_::AlignedData<236> b_89aa8f4e88036b9e = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 71, 108, 111, 98, + 97, 108, 79, 114, 100, 101, 114, 77, + 105, 110, 79, 102, 102, 115, 101, 116, + 115, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 71, 108, 111, 98, + 97, 108, 79, 114, 100, 101, 114, 77, + 97, 120, 79, 102, 102, 115, 101, 116, + 115, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } }; ::capnp::word const* const bp_89aa8f4e88036b9e = b_89aa8f4e88036b9e.words; #if !CAPNP_LITE -static const uint16_t m_89aa8f4e88036b9e[] = {9, 10, 0, 6, 5, 8, 1, 7, 4, 2, 3}; -static const uint16_t i_89aa8f4e88036b9e[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; +static const uint16_t m_89aa8f4e88036b9e[] = {9, 10, 0, 12, 11, 6, 5, 8, 1, 7, 4, 2, 3}; +static const uint16_t i_89aa8f4e88036b9e[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}; const ::capnp::_::RawSchema s_89aa8f4e88036b9e = { - 0x89aa8f4e88036b9e, b_89aa8f4e88036b9e.words, 236, nullptr, m_89aa8f4e88036b9e, - 0, 11, i_89aa8f4e88036b9e, nullptr, nullptr, { &s_89aa8f4e88036b9e, nullptr, nullptr, 0, 0, nullptr }, false + 0x89aa8f4e88036b9e, b_89aa8f4e88036b9e.words, 280, nullptr, m_89aa8f4e88036b9e, + 0, 13, i_89aa8f4e88036b9e, nullptr, nullptr, { &s_89aa8f4e88036b9e, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<107> b_d492b6734d5e3bf5 = { @@ -9253,7 +9401,7 @@ static const ::capnp::_::AlignedData<107> b_d492b6734d5e3bf5 = { 127, 216, 135, 181, 36, 146, 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 232, 107, 0, 0, 23, 111, 0, 0, + 136, 105, 0, 0, 164, 108, 0, 0, 21, 0, 0, 0, 58, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -9375,7 +9523,7 @@ static const ::capnp::_::AlignedData<50> b_bde8ebd7b13d8625 = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 25, 111, 0, 0, 174, 111, 0, 0, + 165, 108, 0, 0, 52, 109, 0, 0, 21, 0, 0, 0, 2, 1, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -9436,7 +9584,7 @@ static const ::capnp::_::AlignedData<55> b_a736c51d292ca752 = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 178, 111, 0, 0, 6, 112, 0, 0, + 54, 109, 0, 0, 135, 109, 0, 0, 21, 0, 0, 0, 50, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -9502,7 +9650,7 @@ static const ::capnp::_::AlignedData<49> b_cd8abc9dabc4b03f = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 10, 112, 0, 0, 215, 112, 0, 0, + 137, 109, 0, 0, 79, 110, 0, 0, 21, 0, 0, 0, 2, 1, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -9562,7 +9710,7 @@ static const ::capnp::_::AlignedData<56> b_cfea684b4bcd0721 = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 219, 112, 0, 0, 98, 113, 0, 0, + 81, 110, 0, 0, 210, 110, 0, 0, 21, 0, 0, 0, 146, 1, 0, 0, 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -9632,7 +9780,7 @@ static const ::capnp::_::AlignedData<69> b_aaeeafe1e9f3ea1c = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 102, 113, 0, 0, 61, 114, 0, 0, + 212, 110, 0, 0, 162, 111, 0, 0, 21, 0, 0, 0, 194, 1, 0, 0, 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -9715,7 +9863,7 @@ static const ::capnp::_::AlignedData<56> b_f5a35661031194d2 = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 65, 114, 0, 0, 201, 114, 0, 0, + 164, 111, 0, 0, 38, 112, 0, 0, 21, 0, 0, 0, 98, 1, 0, 0, 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -9785,7 +9933,7 @@ static const ::capnp::_::AlignedData<35> b_e68edfc0939e63df = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 205, 114, 0, 0, 13, 115, 0, 0, + 40, 112, 0, 0, 101, 112, 0, 0, 21, 0, 0, 0, 42, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -9834,7 +9982,7 @@ static const ::capnp::_::AlignedData<56> b_891a70a671f15cf6 = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 17, 115, 0, 0, 151, 115, 0, 0, + 103, 112, 0, 0, 231, 112, 0, 0, 21, 0, 0, 0, 82, 1, 0, 0, 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -9904,7 +10052,7 @@ static const ::capnp::_::AlignedData<57> b_805c080c10c1e959 = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 155, 115, 0, 0, 23, 117, 0, 0, + 233, 112, 0, 0, 93, 114, 0, 0, 21, 0, 0, 0, 90, 1, 0, 0, 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -9976,7 +10124,7 @@ static const ::capnp::_::AlignedData<53> b_83f094010132ff21 = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 27, 117, 0, 0, 107, 118, 0, 0, + 95, 114, 0, 0, 167, 115, 0, 0, 21, 0, 0, 0, 74, 1, 0, 0, 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -10043,7 +10191,7 @@ static const ::capnp::_::AlignedData<70> b_ebe17f59ac9a1df1 = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 111, 118, 0, 0, 24, 119, 0, 0, + 169, 115, 0, 0, 76, 116, 0, 0, 21, 0, 0, 0, 82, 1, 0, 0, 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -10131,7 +10279,7 @@ static const ::capnp::_::AlignedData<50> b_e06f571aa93eb314 = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 28, 119, 0, 0, 157, 119, 0, 0, + 78, 116, 0, 0, 201, 116, 0, 0, 21, 0, 0, 0, 26, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -10196,7 +10344,7 @@ static const ::capnp::_::AlignedData<108> b_9fd8fc2f462b2d06 = { 127, 216, 135, 181, 36, 146, 125, 181, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 161, 119, 0, 0, 237, 120, 0, 0, + 203, 116, 0, 0, 8, 118, 0, 0, 21, 0, 0, 0, 34, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -10315,7 +10463,7 @@ static const ::capnp::_::AlignedData<52> b_8965edf5597ce627 = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 241, 120, 0, 0, 112, 121, 0, 0, + 10, 118, 0, 0, 131, 118, 0, 0, 21, 0, 0, 0, 90, 1, 0, 0, 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -10381,7 +10529,7 @@ static const ::capnp::_::AlignedData<46> b_aac8bf9b5211388b = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 121, 0, 0, 252, 121, 0, 0, + 133, 118, 0, 0, 10, 119, 0, 0, 21, 0, 0, 0, 98, 1, 0, 0, 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -10438,7 +10586,7 @@ static const ::capnp::_::AlignedData<54> b_ca2d4d0bfe4ae5d9 = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 122, 0, 0, 205, 123, 0, 0, + 12, 119, 0, 0, 206, 120, 0, 0, 21, 0, 0, 0, 250, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -10506,7 +10654,7 @@ static const ::capnp::_::AlignedData<66> b_e193f1f45a9f102e = { 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 209, 123, 0, 0, 69, 125, 0, 0, + 208, 120, 0, 0, 56, 122, 0, 0, 21, 0, 0, 0, 226, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -10583,7 +10731,7 @@ static const ::capnp::_::AlignedData<82> b_fafb9c94c6b54ec9 = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, - 73, 125, 0, 0, 168, 127, 0, 0, + 58, 122, 0, 0, 134, 124, 0, 0, 21, 0, 0, 0, 2, 1, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -10679,7 +10827,7 @@ static const ::capnp::_::AlignedData<39> b_ce64eabcdabb02b5 = { 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 172, 127, 0, 0, 165, 128, 0, 0, + 136, 124, 0, 0, 124, 125, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -10732,7 +10880,7 @@ static const ::capnp::_::AlignedData<27> b_f3bb391da5271019 = { 127, 216, 135, 181, 36, 146, 125, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 169, 128, 0, 0, 213, 128, 0, 0, + 126, 125, 0, 0, 167, 125, 0, 0, 21, 0, 0, 0, 234, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -10770,7 +10918,7 @@ static const ::capnp::_::AlignedData<36> b_87f0466598bb29be = { 127, 216, 135, 181, 36, 146, 125, 181, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 217, 128, 0, 0, 113, 129, 0, 0, + 169, 125, 0, 0, 60, 126, 0, 0, 21, 0, 0, 0, 42, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -10820,7 +10968,7 @@ static const ::capnp::_::AlignedData<64> b_ff44627d34d063ab = { 127, 216, 135, 181, 36, 146, 125, 181, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 129, 0, 0, 211, 129, 0, 0, + 62, 126, 0, 0, 152, 126, 0, 0, 21, 0, 0, 0, 2, 1, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -10895,7 +11043,7 @@ static const ::capnp::_::AlignedData<56> b_8de04c65deeb1510 = { 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 215, 129, 0, 0, 93, 130, 0, 0, + 154, 126, 0, 0, 29, 127, 0, 0, 21, 0, 0, 0, 26, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.h b/tiledb/sm/serialization/tiledb-rest.capnp.h index 62611796c64..e5183212859 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.h +++ b/tiledb/sm/serialization/tiledb-rest.capnp.h @@ -1509,7 +1509,7 @@ struct FragmentMetadata { struct GenericTileOffsets; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(cde352fc27e7ca61, 4, 23) + CAPNP_DECLARE_STRUCT_HEADER(cde352fc27e7ca61, 4, 27) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -1526,7 +1526,7 @@ struct FragmentMetadata::GenericTileOffsets { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(89aa8f4e88036b9e, 3, 8) + CAPNP_DECLARE_STRUCT_HEADER(89aa8f4e88036b9e, 3, 10) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -13275,6 +13275,30 @@ class FragmentMetadata::Reader { inline bool hasArraySchemaName() const; inline ::capnp::Text::Reader getArraySchemaName() const; + inline bool hasTileGlobalOrderMinBuffer() const; + inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Reader + getTileGlobalOrderMinBuffer() const; + + inline bool hasTileGlobalOrderMinVarBuffer() const; + inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Reader + getTileGlobalOrderMinVarBuffer() const; + + inline bool hasTileGlobalOrderMaxBuffer() const; + inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Reader + getTileGlobalOrderMaxBuffer() const; + + inline bool hasTileGlobalOrderMaxVarBuffer() const; + inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Reader + getTileGlobalOrderMaxVarBuffer() const; + private: ::capnp::_::StructReader _reader; template @@ -13781,6 +13805,110 @@ class FragmentMetadata::Builder { inline void adoptArraySchemaName(::capnp::Orphan<::capnp::Text>&& value); inline ::capnp::Orphan<::capnp::Text> disownArraySchemaName(); + inline bool hasTileGlobalOrderMinBuffer(); + inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Builder + getTileGlobalOrderMinBuffer(); + inline void setTileGlobalOrderMinBuffer( + ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Reader value); + inline void setTileGlobalOrderMinBuffer( + ::kj::ArrayPtr< + const ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>::Reader> + value); + inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Builder + initTileGlobalOrderMinBuffer(unsigned int size); + inline void adoptTileGlobalOrderMinBuffer( + ::capnp::Orphan<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>> + disownTileGlobalOrderMinBuffer(); + + inline bool hasTileGlobalOrderMinVarBuffer(); + inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Builder + getTileGlobalOrderMinVarBuffer(); + inline void setTileGlobalOrderMinVarBuffer( + ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Reader value); + inline void setTileGlobalOrderMinVarBuffer( + ::kj::ArrayPtr< + const ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>::Reader> + value); + inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Builder + initTileGlobalOrderMinVarBuffer(unsigned int size); + inline void adoptTileGlobalOrderMinVarBuffer( + ::capnp::Orphan<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>> + disownTileGlobalOrderMinVarBuffer(); + + inline bool hasTileGlobalOrderMaxBuffer(); + inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Builder + getTileGlobalOrderMaxBuffer(); + inline void setTileGlobalOrderMaxBuffer( + ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Reader value); + inline void setTileGlobalOrderMaxBuffer( + ::kj::ArrayPtr< + const ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>::Reader> + value); + inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Builder + initTileGlobalOrderMaxBuffer(unsigned int size); + inline void adoptTileGlobalOrderMaxBuffer( + ::capnp::Orphan<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>> + disownTileGlobalOrderMaxBuffer(); + + inline bool hasTileGlobalOrderMaxVarBuffer(); + inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Builder + getTileGlobalOrderMaxVarBuffer(); + inline void setTileGlobalOrderMaxVarBuffer( + ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Reader value); + inline void setTileGlobalOrderMaxVarBuffer( + ::kj::ArrayPtr< + const ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>::Reader> + value); + inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Builder + initTileGlobalOrderMaxVarBuffer(unsigned int size); + inline void adoptTileGlobalOrderMaxVarBuffer( + ::capnp::Orphan<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>> + disownTileGlobalOrderMaxVarBuffer(); + private: ::capnp::_::StructBuilder _builder; template @@ -13873,6 +14001,14 @@ class FragmentMetadata::GenericTileOffsets::Reader { inline ::uint64_t getProcessedConditionsOffsets() const; + inline bool hasTileGlobalOrderMinOffsets() const; + inline ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>::Reader + getTileGlobalOrderMinOffsets() const; + + inline bool hasTileGlobalOrderMaxOffsets() const; + inline ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>::Reader + getTileGlobalOrderMaxOffsets() const; + private: ::capnp::_::StructReader _reader; template @@ -14033,6 +14169,36 @@ class FragmentMetadata::GenericTileOffsets::Builder { inline ::uint64_t getProcessedConditionsOffsets(); inline void setProcessedConditionsOffsets(::uint64_t value); + inline bool hasTileGlobalOrderMinOffsets(); + inline ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>::Builder + getTileGlobalOrderMinOffsets(); + inline void setTileGlobalOrderMinOffsets( + ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>::Reader value); + inline void setTileGlobalOrderMinOffsets( + ::kj::ArrayPtr value); + inline ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>::Builder + initTileGlobalOrderMinOffsets(unsigned int size); + inline void adoptTileGlobalOrderMinOffsets( + ::capnp::Orphan<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>>&& + value); + inline ::capnp::Orphan<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>> + disownTileGlobalOrderMinOffsets(); + + inline bool hasTileGlobalOrderMaxOffsets(); + inline ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>::Builder + getTileGlobalOrderMaxOffsets(); + inline void setTileGlobalOrderMaxOffsets( + ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>::Reader value); + inline void setTileGlobalOrderMaxOffsets( + ::kj::ArrayPtr value); + inline ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>::Builder + initTileGlobalOrderMaxOffsets(unsigned int size); + inline void adoptTileGlobalOrderMaxOffsets( + ::capnp::Orphan<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>>&& + value); + inline ::capnp::Orphan<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>> + disownTileGlobalOrderMaxOffsets(); + private: ::capnp::_::StructBuilder _builder; template @@ -32463,6 +32629,342 @@ FragmentMetadata::Builder::disownArraySchemaName() { _builder.getPointerField(::capnp::bounded<22>() * ::capnp::POINTERS)); } +inline bool FragmentMetadata::Reader::hasTileGlobalOrderMinBuffer() const { + return !_reader.getPointerField(::capnp::bounded<23>() * ::capnp::POINTERS) + .isNull(); +} +inline bool FragmentMetadata::Builder::hasTileGlobalOrderMinBuffer() { + return !_builder.getPointerField(::capnp::bounded<23>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Reader +FragmentMetadata::Reader::getTileGlobalOrderMinBuffer() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>::get(_reader + .getPointerField( + ::capnp::bounded<23>() * + ::capnp::POINTERS)); +} +inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Builder +FragmentMetadata::Builder::getTileGlobalOrderMinBuffer() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>::get(_builder + .getPointerField( + ::capnp::bounded<23>() * + ::capnp::POINTERS)); +} +inline void FragmentMetadata::Builder::setTileGlobalOrderMinBuffer( + ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>:: + set(_builder.getPointerField(::capnp::bounded<23>() * ::capnp::POINTERS), + value); +} +inline void FragmentMetadata::Builder::setTileGlobalOrderMinBuffer( + ::kj::ArrayPtr< + const ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>::Reader> + value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>:: + set(_builder.getPointerField(::capnp::bounded<23>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Builder +FragmentMetadata::Builder::initTileGlobalOrderMinBuffer(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>:: + init( + _builder.getPointerField(::capnp::bounded<23>() * ::capnp::POINTERS), + size); +} +inline void FragmentMetadata::Builder::adoptTileGlobalOrderMinBuffer( + ::capnp::Orphan<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>:: + adopt( + _builder.getPointerField(::capnp::bounded<23>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>> +FragmentMetadata::Builder::disownTileGlobalOrderMinBuffer() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>::disown(_builder + .getPointerField( + ::capnp::bounded<23>() * + ::capnp::POINTERS)); +} + +inline bool FragmentMetadata::Reader::hasTileGlobalOrderMinVarBuffer() const { + return !_reader.getPointerField(::capnp::bounded<24>() * ::capnp::POINTERS) + .isNull(); +} +inline bool FragmentMetadata::Builder::hasTileGlobalOrderMinVarBuffer() { + return !_builder.getPointerField(::capnp::bounded<24>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Reader +FragmentMetadata::Reader::getTileGlobalOrderMinVarBuffer() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>::get(_reader + .getPointerField( + ::capnp::bounded<24>() * + ::capnp::POINTERS)); +} +inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Builder +FragmentMetadata::Builder::getTileGlobalOrderMinVarBuffer() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>::get(_builder + .getPointerField( + ::capnp::bounded<24>() * + ::capnp::POINTERS)); +} +inline void FragmentMetadata::Builder::setTileGlobalOrderMinVarBuffer( + ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>:: + set(_builder.getPointerField(::capnp::bounded<24>() * ::capnp::POINTERS), + value); +} +inline void FragmentMetadata::Builder::setTileGlobalOrderMinVarBuffer( + ::kj::ArrayPtr< + const ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>::Reader> + value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>:: + set(_builder.getPointerField(::capnp::bounded<24>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Builder +FragmentMetadata::Builder::initTileGlobalOrderMinVarBuffer(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>:: + init( + _builder.getPointerField(::capnp::bounded<24>() * ::capnp::POINTERS), + size); +} +inline void FragmentMetadata::Builder::adoptTileGlobalOrderMinVarBuffer( + ::capnp::Orphan<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>:: + adopt( + _builder.getPointerField(::capnp::bounded<24>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>> +FragmentMetadata::Builder::disownTileGlobalOrderMinVarBuffer() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>::disown(_builder + .getPointerField( + ::capnp::bounded<24>() * + ::capnp::POINTERS)); +} + +inline bool FragmentMetadata::Reader::hasTileGlobalOrderMaxBuffer() const { + return !_reader.getPointerField(::capnp::bounded<25>() * ::capnp::POINTERS) + .isNull(); +} +inline bool FragmentMetadata::Builder::hasTileGlobalOrderMaxBuffer() { + return !_builder.getPointerField(::capnp::bounded<25>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Reader +FragmentMetadata::Reader::getTileGlobalOrderMaxBuffer() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>::get(_reader + .getPointerField( + ::capnp::bounded<25>() * + ::capnp::POINTERS)); +} +inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Builder +FragmentMetadata::Builder::getTileGlobalOrderMaxBuffer() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>::get(_builder + .getPointerField( + ::capnp::bounded<25>() * + ::capnp::POINTERS)); +} +inline void FragmentMetadata::Builder::setTileGlobalOrderMaxBuffer( + ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>:: + set(_builder.getPointerField(::capnp::bounded<25>() * ::capnp::POINTERS), + value); +} +inline void FragmentMetadata::Builder::setTileGlobalOrderMaxBuffer( + ::kj::ArrayPtr< + const ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>::Reader> + value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>:: + set(_builder.getPointerField(::capnp::bounded<25>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Builder +FragmentMetadata::Builder::initTileGlobalOrderMaxBuffer(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>:: + init( + _builder.getPointerField(::capnp::bounded<25>() * ::capnp::POINTERS), + size); +} +inline void FragmentMetadata::Builder::adoptTileGlobalOrderMaxBuffer( + ::capnp::Orphan<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>:: + adopt( + _builder.getPointerField(::capnp::bounded<25>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>> +FragmentMetadata::Builder::disownTileGlobalOrderMaxBuffer() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>::disown(_builder + .getPointerField( + ::capnp::bounded<25>() * + ::capnp::POINTERS)); +} + +inline bool FragmentMetadata::Reader::hasTileGlobalOrderMaxVarBuffer() const { + return !_reader.getPointerField(::capnp::bounded<26>() * ::capnp::POINTERS) + .isNull(); +} +inline bool FragmentMetadata::Builder::hasTileGlobalOrderMaxVarBuffer() { + return !_builder.getPointerField(::capnp::bounded<26>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Reader +FragmentMetadata::Reader::getTileGlobalOrderMaxVarBuffer() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>::get(_reader + .getPointerField( + ::capnp::bounded<26>() * + ::capnp::POINTERS)); +} +inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Builder +FragmentMetadata::Builder::getTileGlobalOrderMaxVarBuffer() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>::get(_builder + .getPointerField( + ::capnp::bounded<26>() * + ::capnp::POINTERS)); +} +inline void FragmentMetadata::Builder::setTileGlobalOrderMaxVarBuffer( + ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>:: + set(_builder.getPointerField(::capnp::bounded<26>() * ::capnp::POINTERS), + value); +} +inline void FragmentMetadata::Builder::setTileGlobalOrderMaxVarBuffer( + ::kj::ArrayPtr< + const ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>::Reader> + value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>:: + set(_builder.getPointerField(::capnp::bounded<26>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>::Builder +FragmentMetadata::Builder::initTileGlobalOrderMaxVarBuffer(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>:: + init( + _builder.getPointerField(::capnp::bounded<26>() * ::capnp::POINTERS), + size); +} +inline void FragmentMetadata::Builder::adoptTileGlobalOrderMaxVarBuffer( + ::capnp::Orphan<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>:: + adopt( + _builder.getPointerField(::capnp::bounded<26>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>> +FragmentMetadata::Builder::disownTileGlobalOrderMaxVarBuffer() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::uint8_t, ::capnp::Kind::PRIMITIVE>, + ::capnp::Kind::LIST>>::disown(_builder + .getPointerField( + ::capnp::bounded<26>() * + ::capnp::POINTERS)); +} + inline ::uint64_t FragmentMetadata::GenericTileOffsets::Reader::getRtree() const { return _reader.getDataField<::uint64_t>( @@ -33003,6 +33505,140 @@ FragmentMetadata::GenericTileOffsets::Builder::setProcessedConditionsOffsets( ::capnp::bounded<2>() * ::capnp::ELEMENTS, value); } +inline bool +FragmentMetadata::GenericTileOffsets::Reader::hasTileGlobalOrderMinOffsets() + const { + return !_reader.getPointerField(::capnp::bounded<8>() * ::capnp::POINTERS) + .isNull(); +} +inline bool +FragmentMetadata::GenericTileOffsets::Builder::hasTileGlobalOrderMinOffsets() { + return !_builder.getPointerField(::capnp::bounded<8>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>::Reader +FragmentMetadata::GenericTileOffsets::Reader::getTileGlobalOrderMinOffsets() + const { + return ::capnp::_:: + PointerHelpers<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>>::get( + _reader.getPointerField(::capnp::bounded<8>() * ::capnp::POINTERS)); +} +inline ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>::Builder +FragmentMetadata::GenericTileOffsets::Builder::getTileGlobalOrderMinOffsets() { + return ::capnp::_:: + PointerHelpers<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>>::get( + _builder.getPointerField(::capnp::bounded<8>() * ::capnp::POINTERS)); +} +inline void +FragmentMetadata::GenericTileOffsets::Builder::setTileGlobalOrderMinOffsets( + ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>::Reader value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>>::set( + _builder.getPointerField(::capnp::bounded<8>() * ::capnp::POINTERS), + value); +} +inline void +FragmentMetadata::GenericTileOffsets::Builder::setTileGlobalOrderMinOffsets( + ::kj::ArrayPtr value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>>::set( + _builder.getPointerField(::capnp::bounded<8>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>::Builder +FragmentMetadata::GenericTileOffsets::Builder::initTileGlobalOrderMinOffsets( + unsigned int size) { + return ::capnp::_:: + PointerHelpers<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>>::init( + _builder.getPointerField(::capnp::bounded<8>() * ::capnp::POINTERS), + size); +} +inline void +FragmentMetadata::GenericTileOffsets::Builder::adoptTileGlobalOrderMinOffsets( + ::capnp::Orphan<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>>&& + value) { + ::capnp::_::PointerHelpers< + ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>>:: + adopt( + _builder.getPointerField(::capnp::bounded<8>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>> +FragmentMetadata::GenericTileOffsets::Builder:: + disownTileGlobalOrderMinOffsets() { + return ::capnp::_::PointerHelpers< + ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>>:: + disown( + _builder.getPointerField(::capnp::bounded<8>() * ::capnp::POINTERS)); +} + +inline bool +FragmentMetadata::GenericTileOffsets::Reader::hasTileGlobalOrderMaxOffsets() + const { + return !_reader.getPointerField(::capnp::bounded<9>() * ::capnp::POINTERS) + .isNull(); +} +inline bool +FragmentMetadata::GenericTileOffsets::Builder::hasTileGlobalOrderMaxOffsets() { + return !_builder.getPointerField(::capnp::bounded<9>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>::Reader +FragmentMetadata::GenericTileOffsets::Reader::getTileGlobalOrderMaxOffsets() + const { + return ::capnp::_:: + PointerHelpers<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>>::get( + _reader.getPointerField(::capnp::bounded<9>() * ::capnp::POINTERS)); +} +inline ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>::Builder +FragmentMetadata::GenericTileOffsets::Builder::getTileGlobalOrderMaxOffsets() { + return ::capnp::_:: + PointerHelpers<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>>::get( + _builder.getPointerField(::capnp::bounded<9>() * ::capnp::POINTERS)); +} +inline void +FragmentMetadata::GenericTileOffsets::Builder::setTileGlobalOrderMaxOffsets( + ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>::Reader value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>>::set( + _builder.getPointerField(::capnp::bounded<9>() * ::capnp::POINTERS), + value); +} +inline void +FragmentMetadata::GenericTileOffsets::Builder::setTileGlobalOrderMaxOffsets( + ::kj::ArrayPtr value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>>::set( + _builder.getPointerField(::capnp::bounded<9>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>::Builder +FragmentMetadata::GenericTileOffsets::Builder::initTileGlobalOrderMaxOffsets( + unsigned int size) { + return ::capnp::_:: + PointerHelpers<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>>::init( + _builder.getPointerField(::capnp::bounded<9>() * ::capnp::POINTERS), + size); +} +inline void +FragmentMetadata::GenericTileOffsets::Builder::adoptTileGlobalOrderMaxOffsets( + ::capnp::Orphan<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>>&& + value) { + ::capnp::_::PointerHelpers< + ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>>:: + adopt( + _builder.getPointerField(::capnp::bounded<9>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>> +FragmentMetadata::GenericTileOffsets::Builder:: + disownTileGlobalOrderMaxOffsets() { + return ::capnp::_::PointerHelpers< + ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>>:: + disown( + _builder.getPointerField(::capnp::bounded<9>() * ::capnp::POINTERS)); +} + inline ::uint64_t MultiPartUploadState::Reader::getPartNumber() const { return _reader.getDataField<::uint64_t>( ::capnp::bounded<0>() * ::capnp::ELEMENTS); diff --git a/tiledb/sm/tile/tile_metadata_generator.cc b/tiledb/sm/tile/tile_metadata_generator.cc index 291f3174e25..4df38a68fe6 100644 --- a/tiledb/sm/tile/tile_metadata_generator.cc +++ b/tiledb/sm/tile/tile_metadata_generator.cc @@ -275,12 +275,17 @@ TileMetadataGenerator::TileMetadataGenerator( const bool var_size, const uint64_t cell_size, const uint64_t cell_val_num) - : var_size_(var_size) + : is_dim_(is_dim) + , var_size_(var_size) , type_(type) , min_(nullptr) , min_size_(0) , max_(nullptr) , max_size_(0) + , global_order_min_(nullptr) + , global_order_min_size_(0) + , global_order_max_(nullptr) + , global_order_max_size_(0) , sum_(sizeof(uint64_t)) , null_count_(0) , cell_size_(cell_size) @@ -375,7 +380,17 @@ void TileMetadataGenerator::process_cell_slab( } void TileMetadataGenerator::set_tile_metadata(WriterTileTuple& tile) { - tile.set_metadata(min_, min_size_, max_, max_size_, sum_, null_count_); + tile.set_metadata( + min_, + min_size_, + max_, + max_size_, + global_order_min_, + global_order_min_size_, + global_order_max_, + global_order_max_size_, + sum_, + null_count_); } /* ****************************** */ @@ -499,6 +514,13 @@ void TileMetadataGenerator::process_cell_range( min_max(fixed_tile, start, end); } + if (is_dim_) { + iassert(end > start); + global_order_min_ = &tile.fixed_tile().data_as()[start]; + global_order_max_ = &tile.fixed_tile().data_as()[end - 1]; + global_order_min_size_ = global_order_max_size_ = sizeof(T); + } + if (has_sum_) { Sum::sum_type>::sum( fixed_tile, start, end, sum_); @@ -517,6 +539,23 @@ void TileMetadataGenerator::process_cell_range( } } + if (is_dim_) { + /* + uint64_t* max_sizes = reinterpret_cast( + loaded_metadata_ptr_->tile_global_order_min_buffer()[dim].data()); + + const uint64_t fixed_offset = tile / sizeof(uint64_t); + max_sizes[fixed_offset] = + data.var_tile().size() - source_offsets[data.cell_num() - 1]; + if (data.cell_num() == 1) { + min_sizes[fixed_offset] = max_sizes[fixed_offset]; + } else { + min_sizes[fixed_offset] = source_offsets[1] - source_offsets[0]; + } + */ + throw std::logic_error("TODO"); + } + if (has_sum_) { Sum::sum_type>::sum_nullable( fixed_tile, validity_tile, start, end, sum_); @@ -531,16 +570,30 @@ void TileMetadataGenerator::process_cell_range_var( const auto& offset_tile = tile.offset_tile(); const auto& var_tile = tile.var_tile(); - // Handle empty tile. - if (!has_min_max_ || offset_tile.size() == 0) { - return; - } - // Get pointers to the data and cell num. auto offset_value = offset_tile.data_as() + start; auto var_data = var_tile.data_as(); auto cell_num = tile.cell_num(); + if (is_dim_) { + iassert(end > start); + global_order_min_ = var_tile.data(); + global_order_min_size_ = + (start == cell_num - 1 ? (var_tile.size() - offset_value[0]) : + (offset_value[1] - offset_value[0])); + + const uint64_t imax = end - start - 1; + global_order_max_ = var_tile.data_u8() + offset_value[imax]; + global_order_max_size_ = + (end == cell_num ? (var_tile.size() - offset_value[imax]) : + (offset_value[imax + 1] - offset_value[imax])); + } + + // Handle empty tile. + if (!has_min_max_ || offset_tile.size() == 0) { + return; + } + // Var size attribute, non nullable. if (!tile.nullable()) { if (min_ == nullptr) { diff --git a/tiledb/sm/tile/tile_metadata_generator.h b/tiledb/sm/tile/tile_metadata_generator.h index 71ac7575b27..8ec33ac939e 100644 --- a/tiledb/sm/tile/tile_metadata_generator.h +++ b/tiledb/sm/tile/tile_metadata_generator.h @@ -239,6 +239,9 @@ class TileMetadataGenerator { /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** Is this tile for a dimension. */ + bool is_dim_; + /** Is this a var tile. */ bool var_size_; @@ -257,6 +260,20 @@ class TileMetadataGenerator { /** Maximum value size for this tile. */ uint64_t max_size_; + /** Value for this dimension of the tile's global order minimum value */ + const void* global_order_min_; + + /** Size of the value for this dimension of the tile's global order minimum + * value */ + uint64_t global_order_min_size_; + + /** Value for this dimension of the tile's global order maximum value */ + const void* global_order_max_; + + /** Size of the value for this dimension of the tile's global order maximum + * value */ + uint64_t global_order_max_size_; + /** Sum of values. */ ByteVec sum_; diff --git a/tiledb/sm/tile/writer_tile_tuple.cc b/tiledb/sm/tile/writer_tile_tuple.cc index e6e823ec153..762b1b6fa70 100644 --- a/tiledb/sm/tile/writer_tile_tuple.cc +++ b/tiledb/sm/tile/writer_tile_tuple.cc @@ -92,6 +92,10 @@ void WriterTileTuple::set_metadata( const uint64_t min_size, const void* max, const uint64_t max_size, + const void* global_order_min, + const uint64_t global_order_min_size, + const void* global_order_max, + const uint64_t global_order_max_size, const ByteVec& sum, const uint64_t null_count) { min_.resize(min_size); @@ -106,6 +110,16 @@ void WriterTileTuple::set_metadata( memcpy(max_.data(), max, max_size); } + if (global_order_min != nullptr) { + global_order_min_.emplace(global_order_min_size); + memcpy(global_order_min_->data(), global_order_min, global_order_min_size); + } + + if (global_order_max != nullptr) { + global_order_max_.emplace(global_order_max_size); + memcpy(global_order_max_->data(), global_order_max, global_order_max_size); + } + sum_ = sum; null_count_ = null_count; diff --git a/tiledb/sm/tile/writer_tile_tuple.h b/tiledb/sm/tile/writer_tile_tuple.h index 37339ecc836..06c625e79d0 100644 --- a/tiledb/sm/tile/writer_tile_tuple.h +++ b/tiledb/sm/tile/writer_tile_tuple.h @@ -148,6 +148,22 @@ class WriterTileTuple { return max_; } + /** + * @return if this is a dimension tile, the value of this dimension for the + * global order minimum coordinate + */ + inline const std::optional& global_order_min() const { + return global_order_min_; + } + + /** + * @return if this is a dimension tile, the value of this dimension for the + * global order maximum coordinate + */ + inline const std::optional& global_order_max() const { + return global_order_max_; + } + /** * Returns the tile null count. * @@ -173,6 +189,12 @@ class WriterTileTuple { * @param min_size Minimum size. * @param max Maximum. * @param max_size Maxmum size. + * @param global_order_min The value of this dimension for the tile's global + * order minimum coordinate + * @param global_order_min_size The size of the above + * @param global_order_max The value of this dimension for the tile's global + * order maximum coordinate + * @param global_order_max_size The size of the above * @param sum Sum. * @param null_count Null count. */ @@ -181,6 +203,10 @@ class WriterTileTuple { const uint64_t min_size, const void* max, const uint64_t max_size, + const void* global_order_min, + const uint64_t global_order_min_size, + const void* global_order_max, + const uint64_t global_order_max_size, const ByteVec& sum, const uint64_t null_count); @@ -256,6 +282,14 @@ class WriterTileTuple { /** Maximum value size for this tile. */ uint64_t max_size_; + /** Value of this dimension in the global order minimum coordinate of this + * tile */ + std::optional global_order_min_; + + /** Value of this dimension in the global order maximum coordinate of this + * tile */ + std::optional global_order_max_; + /** Sum of values. */ ByteVec sum_;