diff --git a/test/src/unit-sparse-global-order-reader.cc b/test/src/unit-sparse-global-order-reader.cc index f41448a57c4..0a7a519f076 100644 --- a/test/src/unit-sparse-global-order-reader.cc +++ b/test/src/unit-sparse-global-order-reader.cc @@ -189,7 +189,7 @@ struct FxRun1D { if (subarray.empty()) { return true; } else { - const CoordType coord = fragment.dim_[record]; + const CoordType coord = fragment.dimension()[record]; for (const auto& range : subarray) { if (range.contains(coord)) { return true; @@ -349,7 +349,7 @@ struct FxRun2D { if (subarray.empty() && !condition.has_value()) { return true; } else { - const int r = fragment.d1_[record], c = fragment.d2_[record]; + const int r = fragment.d1()[record], c = fragment.d2()[record]; for (const auto& range : subarray) { if (range.first.has_value() && !range.first->contains(r)) { continue; @@ -723,36 +723,11 @@ void CSparseGlobalOrderFx::write_fragment( } CApiArray& array = *existing; + Context cppctx = vfs_test_setup_.ctx(); + Array cpparray(cppctx, array, false); - // Create the query. - tiledb_query_t* query; - auto rc = tiledb_query_alloc(context(), array, TILEDB_WRITE, &query); - ASSERTER(rc == TILEDB_OK); - rc = tiledb_query_set_layout(context(), query, TILEDB_UNORDERED); - ASSERTER(rc == TILEDB_OK); - - auto field_sizes = templates::query::make_field_sizes(fragment); - templates::query::set_fields( - context(), - query, - field_sizes, - fragment, - [](unsigned d) { return "d" + std::to_string(d + 1); }, - [](unsigned a) { return "a" + std::to_string(a + 1); }); - - // Submit query. - rc = tiledb_query_submit(context(), query); - ASSERTER(std::optional() == error_if_any(rc)); - - // check that sizes match what we expect - const uint64_t expect_num_cells = fragment.size(); - const uint64_t num_cells = - templates::query::num_cells(fragment, field_sizes); - - ASSERTER(num_cells == expect_num_cells); - - // Clean up. - tiledb_query_free(&query); + templates::query::write_fragment( + fragment, cpparray, TILEDB_UNORDERED); } void CSparseGlobalOrderFx::write_1d_fragment_strings( @@ -1433,23 +1408,23 @@ void CSparseGlobalOrderFx::instance_fragment_skew( // Write a fragment F0 with unique coordinates InstanceType::FragmentType fragment0; - fragment0.dim_.resize(fragment_size); - std::iota(fragment0.dim_.begin(), fragment0.dim_.end(), 1); + fragment0.dimension().resize(fragment_size); + std::iota(fragment0.dimension().begin(), fragment0.dimension().end(), 1); // Write a fragment F1 with lots of duplicates // [100,100,100,100,100,101,101,101,101,101,102,102,102,102,102,...] InstanceType::FragmentType fragment1; - fragment1.dim_.resize(fragment0.dim_.num_cells()); - for (size_t i = 0; i < fragment1.dim_.num_cells(); i++) { - fragment1.dim_[i] = - static_cast((i / 10) + (fragment0.dim_.num_cells() / 2)); + fragment1.dimension().resize(fragment0.dimension().num_cells()); + for (size_t i = 0; i < fragment1.dimension().num_cells(); i++) { + fragment1.dimension()[i] = + static_cast((i / 10) + (fragment0.dimension().num_cells() / 2)); } // atts are whatever, used just for query condition and correctness check auto& f0atts = std::get<0>(fragment0.atts_); - f0atts.resize(fragment0.dim_.num_cells()); + f0atts.resize(fragment0.dimension().num_cells()); std::iota(f0atts.begin(), f0atts.end(), 0); - for (uint64_t i = 0; i < fragment0.dim_.num_cells(); i++) { + for (uint64_t i = 0; i < fragment0.dimension().num_cells(); i++) { if ((i * i) % 7 == 0) { std::get<1>(fragment0.atts_).push_back(std::nullopt); } else { @@ -1462,9 +1437,10 @@ void CSparseGlobalOrderFx::instance_fragment_skew( } auto& f1atts = std::get<0>(fragment1.atts_); - f1atts.resize(fragment1.dim_.num_cells()); - std::iota(f1atts.begin(), f1atts.end(), int(fragment0.dim_.num_cells())); - for (uint64_t i = 0; i < fragment1.dim_.num_cells(); i++) { + f1atts.resize(fragment1.dimension().num_cells()); + std::iota( + f1atts.begin(), f1atts.end(), int(fragment0.dimension().num_cells())); + for (uint64_t i = 0; i < fragment1.dimension().num_cells(); i++) { if ((i * i) % 11 == 0) { std::get<1>(fragment1.atts_).push_back(std::nullopt); } else { @@ -1570,25 +1546,25 @@ void CSparseGlobalOrderFx::instance_fragment_interleave( templates::Fragment1D fragment1; // Write a fragment F0 with tiles [1,3][3,5][5,7][7,9]... - fragment0.dim_.resize(fragment_size); - fragment0.dim_[0] = 1; - for (size_t i = 1; i < fragment0.dim_.num_cells(); i++) { - fragment0.dim_[i] = static_cast(1 + 2 * ((i + 1) / 2)); + fragment0.dimension().resize(fragment_size); + fragment0.dimension()[0] = 1; + for (size_t i = 1; i < fragment0.dimension().num_cells(); i++) { + fragment0.dimension()[i] = static_cast(1 + 2 * ((i + 1) / 2)); } // Write a fragment F1 with tiles [2,4][4,6][6,8][8,10]... - fragment1.dim_.resize(fragment0.dim_.num_cells()); - for (size_t i = 0; i < fragment1.dim_.num_cells(); i++) { - fragment1.dim_[i] = fragment0.dim_[i] + 1; + fragment1.dimension().resize(fragment0.dimension().num_cells()); + for (size_t i = 0; i < fragment1.dimension().num_cells(); i++) { + fragment1.dimension()[i] = fragment0.dimension()[i] + 1; } // atts don't really matter auto& f0atts = std::get<0>(fragment0.atts_); - f0atts.resize(fragment0.dim_.num_cells()); + f0atts.resize(fragment0.dimension().num_cells()); std::iota(f0atts.begin(), f0atts.end(), 0); auto& f1atts = std::get<0>(fragment1.atts_); - f1atts.resize(fragment1.dim_.num_cells()); + f1atts.resize(fragment1.dimension().num_cells()); std::iota(f1atts.begin(), f1atts.end(), int(f0atts.num_cells())); FxRun1D instance; @@ -1694,10 +1670,10 @@ void CSparseGlobalOrderFx::instance_fragment_wide_overlap( for (size_t f = 0; f < num_fragments; f++) { templates::Fragment1D fragment; - fragment.dim_.resize(fragment_size); + fragment.dimension().resize(fragment_size); std::iota( - fragment.dim_.begin(), - fragment.dim_.end(), + fragment.dimension().begin(), + fragment.dimension().end(), instance.array.dimension_.domain.lower_bound + static_cast(f)); auto& atts = std::get<0>(fragment.atts_); @@ -1825,10 +1801,10 @@ void CSparseGlobalOrderFx::instance_merge_bound_duplication( for (size_t f = 0; f < num_fragments; f++) { templates::Fragment1D fragment; - fragment.dim_.resize(fragment_size); + fragment.dimension().resize(fragment_size); std::iota( - fragment.dim_.begin(), - fragment.dim_.end(), + fragment.dimension().begin(), + fragment.dimension().end(), static_cast(f * (fragment_size - 1))); auto& atts = std::get<0>(fragment.atts_); @@ -2014,13 +1990,13 @@ void CSparseGlobalOrderFx::instance_out_of_order_mbrs( for (size_t f = 0; f < num_fragments; f++) { templates::Fragment2D fdata; - fdata.d1_.reserve(fragment_size); - fdata.d2_.reserve(fragment_size); + fdata.d1().reserve(fragment_size); + fdata.d2().reserve(fragment_size); std::get<0>(fdata.atts_).reserve(fragment_size); for (size_t i = 0; i < fragment_size; i++) { - fdata.d1_.push_back(row(f, i)); - fdata.d2_.push_back(col(f, i)); + fdata.d1().push_back(row(f, i)); + fdata.d2().push_back(col(f, i)); std::get<0>(fdata.atts_) .push_back(static_cast(f * fragment_size + i)); } @@ -2221,34 +2197,34 @@ void CSparseGlobalOrderFx::instance_fragment_skew_2d_merge_bound( const int tcol = instance.d2.domain.lower_bound + static_cast(f * instance.d2.extent); for (int i = 0; i < instance.d1.extent * instance.d2.extent - 2; i++) { - fdata.d1_.push_back(trow + i / instance.d1.extent); - fdata.d2_.push_back(tcol + i % instance.d1.extent); + fdata.d1().push_back(trow + i / instance.d1.extent); + fdata.d2().push_back(tcol + i % instance.d1.extent); std::get<0>(fdata.atts_).push_back(att++); } // then some sparse coords in the next space tile, // fill the data tile (if the capacity is 4), we'll call it T - fdata.d1_.push_back(trow); - fdata.d2_.push_back(tcol + instance.d2.extent); + fdata.d1().push_back(trow); + fdata.d2().push_back(tcol + instance.d2.extent); std::get<0>(fdata.atts_).push_back(att++); - fdata.d1_.push_back(trow + instance.d1.extent - 1); - fdata.d2_.push_back(tcol + instance.d2.extent + 2); + fdata.d1().push_back(trow + instance.d1.extent - 1); + fdata.d2().push_back(tcol + instance.d2.extent + 2); std::get<0>(fdata.atts_).push_back(att++); // then begin a new data tile "Tnext" which straddles the bounds of that // space tile. this will have a low MBR. - fdata.d1_.push_back(trow + instance.d1.extent - 1); - fdata.d2_.push_back(tcol + instance.d2.extent + 3); + fdata.d1().push_back(trow + instance.d1.extent - 1); + fdata.d2().push_back(tcol + instance.d2.extent + 3); std::get<0>(fdata.atts_).push_back(att++); - fdata.d1_.push_back(trow); - fdata.d2_.push_back(tcol + 2 * instance.d2.extent); + fdata.d1().push_back(trow); + fdata.d2().push_back(tcol + 2 * instance.d2.extent); std::get<0>(fdata.atts_).push_back(att++); // then add a point P which is less than the lower bound of Tnext's MBR, // and also between the last two coordinates of T FxRun2D::FragmentType fpoint; - fpoint.d1_.push_back(trow + instance.d1.extent - 1); - fpoint.d2_.push_back(tcol + instance.d1.extent + 1); + fpoint.d1().push_back(trow + instance.d1.extent - 1); + fpoint.d2().push_back(tcol + instance.d1.extent + 1); std::get<0>(fpoint.atts_).push_back(att++); instance.fragments.push_back(fdata); @@ -2370,13 +2346,13 @@ void CSparseGlobalOrderFx::instance_fragment_full_copy_1d( for (size_t f = 0; f < num_fragments; f++) { FxRunType::FragmentType fragment; - fragment.dim_.resize(fragment_size); + fragment.dimension().resize(fragment_size); std::iota( - fragment.dim_.begin(), - fragment.dim_.end(), + fragment.dimension().begin(), + fragment.dimension().end(), dimension.domain.lower_bound); - std::get<0>(fragment.atts_).resize(fragment.dim_.num_cells()); + std::get<0>(fragment.atts_).resize(fragment.dimension().num_cells()); std::iota( std::get<0>(fragment.atts_).begin(), std::get<0>(fragment.atts_).end(), @@ -3324,8 +3300,8 @@ void CSparseGlobalOrderFx::instance_repeatable_read_submillisecond( for (uint64_t t = 0; t < fragment_same_timestamp_runs.size(); t++) { for (uint64_t f = 0; f < fragment_same_timestamp_runs[t]; f++) { FxRun2D::FragmentType fragment; - fragment.d1_ = {1, 2 + static_cast(t)}; - fragment.d2_ = {1, 2 + static_cast(f)}; + fragment.d1() = {1, 2 + static_cast(t)}; + fragment.d2() = {1, 2 + static_cast(f)}; std::get<0>(fragment.atts_) = std::vector{ static_cast(instance.fragments.size()), static_cast(instance.fragments.size())}; @@ -3354,7 +3330,7 @@ void CSparseGlobalOrderFx::instance_repeatable_read_submillisecond( CApiArray array(context(), raw_array, TILEDB_WRITE); for (uint64_t f = 0; f < fragment_same_timestamp_runs[t]; f++, i++) { - write_fragment( + write_fragment( instance.fragments[i], &array); } } @@ -3448,64 +3424,15 @@ TEST_CASE_METHOD( */ template void CSparseGlobalOrderFx::create_array(const Instance& instance) { - const auto dimensions = instance.dimensions(); - const auto attributes = instance.attributes(); - - std::vector dimension_names; - std::vector dimension_types; - std::vector dimension_ranges; - std::vector dimension_extents; - auto add_dimension = [&]( - const templates::Dimension& dimension) { - using CoordType = templates::Dimension::value_type; - dimension_names.push_back("d" + std::to_string(dimension_names.size() + 1)); - dimension_types.push_back(static_cast(D)); - dimension_ranges.push_back( - const_cast(&dimension.domain.lower_bound)); - dimension_extents.push_back(const_cast(&dimension.extent)); - }; - std::apply( - [&](const templates::Dimension&... dimension) { - (add_dimension(dimension), ...); - }, - dimensions); - - std::vector attribute_names; - std::vector attribute_types; - std::vector attribute_cell_val_nums; - std::vector attribute_nullables; - std::vector> attribute_compressors; - auto add_attribute = [&](Datatype datatype, - uint32_t cell_val_num, - bool nullable) { - attribute_names.push_back("a" + std::to_string(attribute_names.size() + 1)); - attribute_types.push_back(static_cast(datatype)); - attribute_cell_val_nums.push_back(cell_val_num); - attribute_nullables.push_back(nullable); - attribute_compressors.push_back(std::make_pair(TILEDB_FILTER_NONE, -1)); - }; - for (const auto& [datatype, cell_val_num, nullable] : attributes) { - add_attribute(datatype, cell_val_num, nullable); - } - - tiledb::test::create_array( - context(), + templates::ddl::create_array( array_name_, - TILEDB_SPARSE, - dimension_names, - dimension_types, - dimension_ranges, - dimension_extents, - attribute_names, - attribute_types, - attribute_cell_val_nums, - attribute_compressors, + Context(context(), false), + instance.dimensions(), + instance.attributes(), instance.tile_order(), instance.cell_order(), instance.tile_capacity(), - instance.allow_duplicates(), - false, - {attribute_nullables}); + instance.allow_duplicates()); } /** @@ -3534,13 +3461,13 @@ DeleteArrayGuard CSparseGlobalOrderFx::run_create(Instance& instance) { // the tile extent is 2 // create_default_array_1d(instance.array); - create_array(instance); + create_array(instance); DeleteArrayGuard arrayguard(context(), array_name_.c_str()); // write all fragments for (auto& fragment : instance.fragments) { - write_fragment(fragment); + write_fragment(fragment); } return arrayguard; @@ -3550,7 +3477,7 @@ template void CSparseGlobalOrderFx::run_execute(Instance& instance) { ASSERTER(instance.num_user_cells > 0); - std::decay_t expect; + std::decay_t expect; // for de-duplicating, track the fragment that each coordinate came from // we will use this to select the coordinate from the most recent fragment @@ -3779,19 +3706,7 @@ void CSparseGlobalOrderFx::run_execute(Instance& instance) { ASSERTER(num_cells == num_cells_bound); } - std::apply( - [&](auto&... field) { - std::apply( - [&](const auto&... field_cursor) { - std::apply( - [&](const auto&... field_size) { - (field.apply_cursor(field_cursor, field_size), ...); - }, - field_sizes); - }, - outcursor); - }, - std::tuple_cat(outdims, outatts)); + templates::query::apply_cursor(out, outcursor, field_sizes); const uint64_t cursor_cells = templates::query::num_cells(out, outcursor); @@ -4128,11 +4043,11 @@ void show(const FxRun2D& instance, std::ostream& os) { os << "\t\t{" << std::endl; os << "\t\t\t\"d1\": [" << std::endl; os << "\t\t\t\t"; - show(fragment.d1_, os); + show(fragment.d1(), os); os << std::endl; os << "\t\t\t\"d2\": [" << std::endl; os << "\t\t\t\t"; - show(fragment.d2_, os); + show(fragment.d2(), os); os << std::endl; os << "\t\t\t], " << std::endl; os << "\t\t\t\"atts\": [" << std::endl; diff --git a/test/support/rapidcheck/array_templates.h b/test/support/rapidcheck/array_templates.h index 41f5ac82495..f2c1dacc232 100644 --- a/test/support/rapidcheck/array_templates.h +++ b/test/support/rapidcheck/array_templates.h @@ -139,7 +139,19 @@ Gen make_coordinate(const templates::Domain& domain) { // whereas the domain upper bound is inclusive. // As a result some contortion is required to deal // with numeric_limits. - if (std::is_signed::value) { + if constexpr (std::is_same_v) { + // NB: poor performance with small domains for sure + return gen::suchThat( + gen::map( + gen::string(), + [](std::string s) { + StringDimensionCoordType v(s.begin(), s.end()); + return v; + }), + [domain](const StringDimensionCoordType& s) { + return domain.lower_bound <= s && s <= domain.upper_bound; + }); + } else if constexpr (std::is_signed::value) { if (int64_t(domain.upper_bound) < std::numeric_limits::max()) { return gen::cast(gen::inRange( int64_t(domain.lower_bound), int64_t(domain.upper_bound + 1))); @@ -185,7 +197,11 @@ Gen> make_fragment_1d( std::apply( [&](std::vector tup_d1, auto... tup_atts) { - coords.values_ = tup_d1; + if constexpr (std::is_same_v) { + coords = query_buffers(tup_d1); + } else { + coords.values_ = tup_d1; + } atts = std::apply( [&](std::vector... att) { return std::make_tuple(query_buffers(att)...); @@ -195,7 +211,7 @@ Gen> make_fragment_1d( stdx::transpose(cells)); return Fragment1D{ - .dim_ = coords, .atts_ = atts}; + std::make_tuple(coords), atts}; }); } @@ -233,13 +249,127 @@ Gen> make_fragment_2d( stdx::transpose(cells)); return Fragment2D{ - .d1_ = coords_d1, .d2_ = coords_d2, .atts_ = atts}; + std::make_tuple(coords_d1, coords_d2), atts}; + }); +} + +template < + DimensionType D1, + DimensionType D2, + DimensionType D3, + AttributeType... Att> +Gen> make_fragment_3d( + bool allow_duplicates, + std::optional> d1, + std::optional> d2, + std::optional> d3) { + auto coord_d1 = + (d1.has_value() ? make_coordinate(d1.value()) : gen::arbitrary()); + auto coord_d2 = + (d2.has_value() ? make_coordinate(d2.value()) : gen::arbitrary()); + auto coord_d3 = + (d3.has_value() ? make_coordinate(d3.value()) : gen::arbitrary()); + + using Cell = std::tuple; + + auto cell = + gen::tuple(coord_d1, coord_d2, coord_d3, gen::arbitrary()...); + + auto uniqueCoords = [](const Cell& cell) { + return std::make_tuple( + std::get<0>(cell), std::get<1>(cell), std::get<2>(cell)); + }; + + auto cells = gen::nonEmpty( + allow_duplicates ? gen::container>(cell) : + gen::uniqueBy>(cell, uniqueCoords)); + + return gen::map(cells, [](std::vector cells) { + std::vector coords_d1; + std::vector coords_d2; + std::vector coords_d3; + std::tuple...> atts; + + std::apply( + [&](std::vector tup_d1, + std::vector tup_d2, + std::vector tup_d3, + auto... tup_atts) { + coords_d1 = tup_d1; + coords_d2 = tup_d2; + coords_d3 = tup_d3; + atts = std::make_tuple(tup_atts...); + }, + stdx::transpose(cells)); + + return Fragment3D{ + std::make_tuple(coords_d1, coords_d2, coords_d3), atts}; }); } -template <> -void show>(const templates::Domain& domain, std::ostream& os) { - os << "[" << domain.lower_bound << ", " << domain.upper_bound << "]"; +void showValue(const templates::Domain& domain, std::ostream& os); +void showValue(const templates::Domain& domain, std::ostream& os); +void showValue(const templates::Domain& domain, std::ostream& os); + +namespace detail { + +/** + * Specialization of `rc::detail::ShowDefault` for `query_buffers` of + * fundamental cell type. + * + * Parameters `A` and `B` are SFINAE which in principle allow less verbose + * alternative paths to providing this custom functionality. + */ +template +struct ShowDefault, A, B> { + static void show(const query_buffers& value, std::ostream& os) { + ::rc::show(value.values_, os); + } +}; + +/** + * Specialization of `rc::detail::ShowDefault` for + * `query_buffers>`. + * + * Parameters `A` and `B` are SFINAE which in principle allow less verbose + * alternative paths to providing this custom functionality. + */ +template +struct ShowDefault>, A, B> { + static void show( + const query_buffers>& value, std::ostream& os) { + std::vector values; + for (uint64_t c = 0; c < value.num_cells(); c++) { + values.push_back(std::string(value[c].begin(), value[c].end())); + } + ::rc::show(values, os); + } +}; + +} // namespace detail + +/** + * Generic logic to for showing a `templates::FragmentType`. + */ +template +void showFragment( + const templates::Fragment& value, + std::ostream& os) { + auto showField = [&](const query_buffers& field) { + os << "\t\t"; + show(field, os); + os << std::endl; + }; + os << "{" << std::endl << "\t\"dimensions\": [" << std::endl; + std::apply( + [&](const auto&... dimension) { (showField(dimension), ...); }, + value.dimensions()); + os << "\t]" << std::endl; + os << "\t\"attributes\": [" << std::endl; + std::apply( + [&](const auto&... attribute) { (showField(attribute), ...); }, + value.attributes()); + os << "\t]" << std::endl << "}" << std::endl; } } // namespace rc diff --git a/test/support/src/array_templates.h b/test/support/src/array_templates.h index 7858077b2b7..bc75487e8ca 100644 --- a/test/support/src/array_templates.h +++ b/test/support/src/array_templates.h @@ -36,12 +36,14 @@ #include "tiledb.h" #include "tiledb/common/unreachable.h" +#include "tiledb/sm/cpp_api/tiledb" #include "tiledb/sm/query/ast/query_ast.h" #include "tiledb/type/datatype_traits.h" #include "tiledb/type/range/range.h" #include #include +#include #include #include #include @@ -57,6 +59,9 @@ class Dimension; namespace tiledb::test::templates { +using StringDimensionCoordType = std::vector; +using StringDimensionCoordView = std::span; + /** * Adapts a `std::tuple` whose fields are all `GlobalCellCmp` * to itself be `GlobalCellCmp`. @@ -67,26 +72,42 @@ struct global_cell_cmp_std_tuple { : tup_(tup) { } + private: + template + static constexpr tiledb::common::UntypedDatumView static_coord_datum( + const T& field) { + static_assert( + stdx::is_fundamental || + std::is_same_v || + std::is_same_v); + if constexpr (stdx::is_fundamental) { + return UntypedDatumView(&field, sizeof(T)); + } else { + return UntypedDatumView(field.data(), field.size()); + } + } + + template + static tiledb::common::UntypedDatumView try_dimension_datum( + const StdTuple& tup, unsigned dim) { + if (dim == I) { + return static_coord_datum(std::get(tup)); + } else if constexpr (I + 1 < std::tuple_size_v) { + return try_dimension_datum(tup, dim); + } else { + // NB: probably not reachable in practice + throw std::logic_error("Out of bounds access to dimension tuple"); + } + } + + public: tiledb::common::UntypedDatumView dimension_datum( const tiledb::sm::Dimension&, unsigned dim_idx) const { - return std::apply( - [&](const auto&... field) { - size_t sizes[] = {sizeof(std::decay_t)...}; - const void* const ptrs[] = { - static_cast(std::addressof(field))...}; - return UntypedDatumView(ptrs[dim_idx], sizes[dim_idx]); - }, - tup_); + return try_dimension_datum<0>(tup_, dim_idx); } const void* coord(unsigned dim) const { - return std::apply( - [&](const auto&... field) { - const void* const ptrs[] = { - static_cast(std::addressof(field))...}; - return ptrs[dim]; - }, - tup_); + return try_dimension_datum<0>(tup_, dim).content(); } StdTuple tup_; @@ -106,11 +127,12 @@ struct query_buffers {}; * Constrains types which can be used as the physical type of a dimension. */ template -concept DimensionType = requires(const D& coord) { - typename std::is_signed; - { coord < coord } -> std::same_as; - { D(int64_t(coord)) } -> std::same_as; -}; +concept DimensionType = + std::is_same_v or requires(const D& coord) { + typename std::is_signed; + { coord < coord } -> std::same_as; + { D(int64_t(coord)) } -> std::same_as; + }; /** * Constrains types which can be used as the physical type of an attribute. @@ -206,6 +228,20 @@ struct Dimension { value_type extent; }; +template <> +struct Dimension { + using value_type = StringDimensionCoordType; + + Dimension() { + } + + Dimension(const Domain& domain) + : domain(domain) { + } + + std::optional> domain; +}; + template struct static_attribute {}; @@ -436,6 +472,10 @@ struct query_buffers { : values_(cells) { } + query_buffers(std::initializer_list cells) + : values_(cells) { + } + bool operator==(const self_type&) const = default; uint64_t num_cells() const { @@ -481,6 +521,11 @@ struct query_buffers { return *this; } + self_type& operator=(const std::initializer_list& values) { + values_ = values; + return *this; + } + query_field_size_type make_field_size(uint64_t cell_limit) const { return sizeof(T) * std::min(cell_limit, values_.size()); } @@ -1125,42 +1170,139 @@ struct query_buffers>> { } }; -/** - * Data for a one-dimensional array - */ -template -struct Fragment1D { - using DimensionType = D; +template +struct Fragment { + private: + template + struct to_query_buffers { + using value_type = std::tuple...>; + using ref_type = std::tuple&...>; + using const_ref_type = std::tuple&...>; + }; + + template + static to_query_buffers::value_type f_qb_value(std::tuple) { + return std::declval::value_type>(); + } + + template + static to_query_buffers::ref_type f_qb_ref(std::tuple) { + return std::declval::ref_type>(); + } + + template + static to_query_buffers::const_ref_type f_qb_const_ref( + std::tuple) { + return std::declval::const_ref_type>(); + } + + template + using value_tuple_query_buffers = decltype(f_qb_value(std::declval())); + + template + using ref_tuple_query_buffers = decltype(f_qb_ref(std::declval())); + + template + using const_ref_tuple_query_buffers = + decltype(f_qb_const_ref(std::declval())); + + public: + using DimensionTuple = _DimensionTuple; + using AttributeTuple = _AttributeTuple; + + using self_type = Fragment; + + using DimensionBuffers = value_tuple_query_buffers; + using DimensionBuffersRef = ref_tuple_query_buffers; + using DimensionBuffersConstRef = + const_ref_tuple_query_buffers; + + using AttributeBuffers = value_tuple_query_buffers; + using AttributeBuffersRef = ref_tuple_query_buffers; + using AttributeBuffersConstRef = + const_ref_tuple_query_buffers; - query_buffers dim_; - std::tuple...> atts_; + DimensionBuffers dims_; + AttributeBuffers atts_; + + uint64_t num_cells() const { + static_assert( + std::tuple_size::value > 0 || + std::tuple_size::value > 0); + + if constexpr (std::tuple_size::value == 0) { + return std::get<0>(atts_).num_cells(); + } else { + return std::get<0>(atts_).num_cells(); + } + } uint64_t size() const { - return dim_.num_cells(); + return num_cells(); + } + + const DimensionBuffersConstRef dimensions() const { + return std::apply( + [](const auto&... field) { return std::forward_as_tuple(field...); }, + dims_); } - std::tuple&> dimensions() const { - return std::tuple&>(dim_); + DimensionBuffersRef dimensions() { + return std::apply( + [](auto&... field) { return std::forward_as_tuple(field...); }, dims_); } - std::tuple&...> attributes() const { + const AttributeBuffersConstRef attributes() const { return std::apply( - [](const query_buffers&... attribute) { - return std::tuple&...>(attribute...); - }, + [](const auto&... field) { return std::forward_as_tuple(field...); }, atts_); } - std::tuple&> dimensions() { - return std::tuple&>(dim_); + AttributeBuffersRef attributes() { + return std::apply( + [](auto&... field) { return std::forward_as_tuple(field...); }, atts_); } - std::tuple&...> attributes() { - return std::apply( - [](query_buffers&... attribute) { - return std::tuple&...>(attribute...); + void reserve(uint64_t num_cells) { + std::apply( + [num_cells](Ts&... field) { + (field.reserve(num_cells), ...); }, - atts_); + std::tuple_cat(dimensions(), attributes())); + } + + void resize(uint64_t num_cells) { + std::apply( + [num_cells](Ts&... field) { + (field.resize(num_cells), ...); + }, + std::tuple_cat(dimensions(), attributes())); + } + + void extend(const self_type& other) { + std::apply( + [&](Ts&... dst) { + std::apply( + [&](const Us&... src) { (dst.extend(src), ...); }, + std::tuple_cat(other.dimensions(), other.attributes())); + }, + std::tuple_cat(dimensions(), attributes())); + } +}; + +/** + * Data for a one-dimensional array + */ +template +struct Fragment1D : public Fragment, std::tuple> { + using DimensionType = D; + + const query_buffers& dimension() const { + return std::get<0>(this->dimensions()); + } + + query_buffers& dimension() { + return std::get<0>(this->dimensions()); } }; @@ -1168,39 +1310,52 @@ struct Fragment1D { * Data for a two-dimensional array */ template -struct Fragment2D { - query_buffers d1_; - query_buffers d2_; - std::tuple...> atts_; +struct Fragment2D : public Fragment, std::tuple> { + const query_buffers& d1() const { + return std::get<0>(this->dimensions()); + } - uint64_t size() const { - return d1_.num_cells(); + const query_buffers& d2() const { + return std::get<1>(this->dimensions()); } - std::tuple&, const query_buffers&> dimensions() - const { - return std::tuple&, const query_buffers&>( - d1_, d2_); + query_buffers& d1() { + return std::get<0>(this->dimensions()); } - std::tuple&, query_buffers&> dimensions() { - return std::tuple&, query_buffers&>(d1_, d2_); + query_buffers& d2() { + return std::get<1>(this->dimensions()); } +}; - std::tuple&...> attributes() const { - return std::apply( - [](const query_buffers&... attribute) { - return std::tuple&...>(attribute...); - }, - atts_); +/** + * Data for a three-dimensional array + */ +template +struct Fragment3D + : public Fragment, std::tuple> { + const query_buffers& d1() const { + return std::get<0>(this->dimensions()); } - std::tuple&...> attributes() { - return std::apply( - [](query_buffers&... attribute) { - return std::tuple&...>(attribute...); - }, - atts_); + const query_buffers& d2() const { + return std::get<1>(this->dimensions()); + } + + const query_buffers& d3() const { + return std::get<2>(this->dimensions()); + } + + query_buffers& d1() { + return std::get<0>(this->dimensions()); + } + + query_buffers& d2() { + return std::get<1>(this->dimensions()); + } + + query_buffers& d3() { + return std::get<2>(this->dimensions()); } }; @@ -1317,10 +1472,12 @@ namespace query { template auto make_field_sizes( F& fragment, uint64_t cell_limit = std::numeric_limits::max()) { + typename F::DimensionBuffersRef dims = fragment.dimensions(); + typename F::AttributeBuffersRef atts = fragment.attributes(); return [cell_limit](std::tuple fields) { return query_applicator::make_field_sizes( fields, cell_limit); - }(std::tuple_cat(fragment.dimensions(), fragment.attributes())); + }(std::tuple_cat(dims, atts)); } template @@ -1328,6 +1485,31 @@ using fragment_field_sizes_t = decltype(make_field_sizes( std::declval(), std::declval())); +/** + * Apply field cursor and sizes to each field of `fragment`. + */ +template +void apply_cursor( + F& fragment, + const fragment_field_sizes_t& cursor, + const fragment_field_sizes_t& field_sizes) { + typename F::DimensionBuffersRef dims = fragment.dimensions(); + typename F::AttributeBuffersRef atts = fragment.attributes(); + std::apply( + [&](auto&... field) { + std::apply( + [&](const auto&... field_cursor) { + std::apply( + [&](const auto&... field_size) { + (field.apply_cursor(field_cursor, field_size), ...); + }, + field_sizes); + }, + cursor); + }, + std::tuple_cat(dims, atts)); +} + /** * Set buffers on `query` for the tuple of field columns */ @@ -1349,24 +1531,31 @@ void set_fields( std::decay_t, std::tuple_size_v>::value(field_cursors); - [&](std::tuple fields) { - query_applicator::set( - ctx, - query, - split_sizes.first, - fields, - dimension_name, - split_cursors.first); - }(fragment.dimensions()); - [&](std::tuple fields) { - query_applicator::set( - ctx, - query, - split_sizes.second, - fields, - attribute_name, - split_cursors.second); - }(fragment.attributes()); + if constexpr (!std:: + is_same_v>) { + [&](std::tuple fields) { + query_applicator::set( + ctx, + query, + split_sizes.first, + fields, + dimension_name, + split_cursors.first); + }(fragment.dimensions()); + } + + if constexpr (!std:: + is_same_v>) { + [&](std::tuple fields) { + query_applicator::set( + ctx, + query, + split_sizes.second, + fields, + attribute_name, + split_cursors.second); + }(fragment.attributes()); + } } /** @@ -1379,8 +1568,126 @@ uint64_t num_cells(const F& fragment, const auto& field_sizes) { }(std::tuple_cat(fragment.dimensions(), fragment.attributes())); } +/** + * Writes a fragment to an array. + */ +template +void write_fragment( + const Fragment& fragment, + Array& forwrite, + tiledb_layout_t layout = TILEDB_UNORDERED) { + Query query(forwrite); + query.set_layout(layout); + + auto field_sizes = + make_field_sizes(const_cast(fragment)); + templates::query::set_fields( + query.ctx().ptr().get(), + query.ptr().get(), + field_sizes, + const_cast(fragment), + [](unsigned d) { return "d" + std::to_string(d + 1); }, + [](unsigned a) { return "a" + std::to_string(a + 1); }); + + const auto status = query.submit(); + ASSERTER(status == Query::Status::COMPLETE); + + if (layout == TILEDB_GLOBAL_ORDER) { + query.finalize(); + } + + // check that sizes match what we expect + const uint64_t expect_num_cells = fragment.size(); + const uint64_t num_cells = + templates::query::num_cells(fragment, field_sizes); + + ASSERTER(num_cells == expect_num_cells); +} + } // namespace query +namespace ddl { + +/** + * Creates an array with a schema whose dimensions and attributes + * come from the simplified arguments. + * The names of the dimensions are d1, d2, etc. + * The names of the attributes are a1, a2, etc. + */ +template +void create_array( + const std::string& array_name, + const Context& context, + const std::tuple&...> dimensions, + std::vector> attributes, + tiledb_layout_t tile_order, + tiledb_layout_t cell_order, + uint64_t tile_capacity, + bool allow_duplicates) { + std::vector dimension_names; + std::vector dimension_types; + std::vector dimension_ranges; + std::vector dimension_extents; + auto add_dimension = [&]( + const templates::Dimension& dimension) { + using CoordType = templates::Dimension::value_type; + dimension_names.push_back("d" + std::to_string(dimension_names.size() + 1)); + dimension_types.push_back(static_cast(D)); + if constexpr (std::is_same_v) { + dimension_ranges.push_back(nullptr); + dimension_extents.push_back(nullptr); + } else { + dimension_ranges.push_back( + const_cast(&dimension.domain.lower_bound)); + dimension_extents.push_back(const_cast(&dimension.extent)); + } + }; + std::apply( + [&](const templates::Dimension&... dimension) { + (add_dimension(dimension), ...); + }, + dimensions); + + std::vector attribute_names; + std::vector attribute_types; + std::vector attribute_cell_val_nums; + std::vector attribute_nullables; + std::vector> attribute_compressors; + auto add_attribute = [&](Datatype datatype, + uint32_t cell_val_num, + bool nullable) { + attribute_names.push_back("a" + std::to_string(attribute_names.size() + 1)); + attribute_types.push_back(static_cast(datatype)); + attribute_cell_val_nums.push_back(cell_val_num); + attribute_nullables.push_back(nullable); + attribute_compressors.push_back(std::make_pair(TILEDB_FILTER_NONE, -1)); + }; + for (const auto& [datatype, cell_val_num, nullable] : attributes) { + add_attribute(datatype, cell_val_num, nullable); + } + + tiledb::test::create_array( + context.ptr().get(), + array_name, + TILEDB_SPARSE, + dimension_names, + dimension_types, + dimension_ranges, + dimension_extents, + attribute_names, + attribute_types, + attribute_cell_val_nums, + attribute_compressors, + tile_order, + cell_order, + tile_capacity, + allow_duplicates, + false, + {attribute_nullables}); +} + +} // namespace ddl + } // namespace tiledb::test::templates #endif diff --git a/tiledb/common/types/untyped_datum.h b/tiledb/common/types/untyped_datum.h index 1f61e681ea7..51ee98a0f7c 100644 --- a/tiledb/common/types/untyped_datum.h +++ b/tiledb/common/types/untyped_datum.h @@ -41,23 +41,23 @@ class UntypedDatumView { size_t datum_size_; public: - UntypedDatumView(const void* content, size_t size) + constexpr UntypedDatumView(const void* content, size_t size) : datum_content_(content) , datum_size_(size) { } - UntypedDatumView(std::string_view ss) + constexpr UntypedDatumView(std::string_view ss) : datum_content_(ss.data()) , datum_size_(ss.size()) { } - [[nodiscard]] inline const void* content() const { + [[nodiscard]] constexpr inline const void* content() const { return datum_content_; } - [[nodiscard]] inline size_t size() const { + [[nodiscard]] constexpr inline size_t size() const { return datum_size_; } template - [[nodiscard]] inline const T& value_as() const { + [[nodiscard]] constexpr inline const T& value_as() const { return *static_cast(datum_content_); } }; diff --git a/tiledb/sm/cpp_api/array.h b/tiledb/sm/cpp_api/array.h index 5c4bca8c272..03c50aefabd 100644 --- a/tiledb/sm/cpp_api/array.h +++ b/tiledb/sm/cpp_api/array.h @@ -319,6 +319,11 @@ class Array { return std::string(uri); } + /** Get the Context for the array. */ + const Context& context() const { + return ctx_.get(); + } + /** Get the ArraySchema for the array. **/ ArraySchema schema() const { auto& ctx = ctx_.get(); diff --git a/tiledb/sm/cpp_api/query.h b/tiledb/sm/cpp_api/query.h index 2e5308180d2..cbb0eafd361 100644 --- a/tiledb/sm/cpp_api/query.h +++ b/tiledb/sm/cpp_api/query.h @@ -171,6 +171,34 @@ class Query { : Query(ctx, array, array.query_type()) { } + /** + * Creates a TileDB query object. + * + * The context and query type (read or write) are inferred from the array + * object, which was opened with a specific query type. + * + * The storage manager also acquires a **shared lock** on the array. This + * means multiple read and write queries to the same array can be made + * concurrently (in TileDB, only consolidation requires an exclusive lock for + * a short period of time). + * + * **Example:** + * + * @code{.cpp} + * // Open the array for writing + * tiledb::Context ctx; + * tiledb::Array array(ctx, "my_array", TILEDB_WRITE); + * Query query(array); + * // Equivalent to: + * // Query query(ctx, array, TILEDB_WRITE); + * @endcode + * + * @param array Open Array object + */ + Query(const Array& array) + : Query(array.context(), array) { + } + Query(const Query&) = default; Query(Query&&) = default; Query& operator=(const Query&) = default;