Skip to content

Commit 37069b2

Browse files
shaunrd0shaunrd0
committed
Add C.41 Subarray constructor
+ Unstatus subarray_from_capnp
1 parent 74ebc44 commit 37069b2

File tree

5 files changed

+197
-109
lines changed

5 files changed

+197
-109
lines changed

‎test/support/src/serialization_wrappers.cc‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -204,11 +204,15 @@ void tiledb_subarray_serialize(
204204
.ok());
205205
// Deserialize
206206
tiledb_subarray_t* deserialized_subarray;
207+
auto layout = (*subarray)->subarray_->layout();
208+
auto stats = (*subarray)->subarray_->stats();
209+
shared_ptr<Logger> dummy_logger = make_shared<Logger>(HERE(), "");
210+
207211
tiledb::test::require_tiledb_ok(
208212
ctx, tiledb_subarray_alloc(ctx, array, &deserialized_subarray));
209-
REQUIRE(tiledb::sm::serialization::subarray_from_capnp(
210-
builder, deserialized_subarray->subarray_)
211-
.ok());
213+
*deserialized_subarray->subarray_ =
214+
tiledb::sm::serialization::subarray_from_capnp(
215+
builder, array->array_.get(), layout, stats, dummy_logger);
212216
*subarray = deserialized_subarray;
213217
#endif
214218
}

‎tiledb/sm/serialization/query.cc‎

Lines changed: 87 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -265,89 +265,112 @@ Status subarray_to_capnp(
265265
return Status::Ok();
266266
}
267267

268-
Status subarray_from_capnp(
269-
const capnp::Subarray::Reader& reader, Subarray* subarray) {
270-
RETURN_NOT_OK(subarray->set_coalesce_ranges(reader.getCoalesceRanges()));
268+
Subarray subarray_from_capnp(
269+
const capnp::Subarray::Reader& reader,
270+
const Array* array,
271+
Layout layout,
272+
stats::Stats* parent_stats,
273+
shared_ptr<Logger> logger) {
274+
bool coalesce_ranges = reader.getCoalesceRanges();
271275
auto ranges_reader = reader.getRanges();
276+
272277
uint32_t dim_num = ranges_reader.size();
278+
std::vector<RangeSetAndSuperset> range_subset(dim_num);
279+
std::vector<bool> is_default(dim_num, false);
273280
for (uint32_t i = 0; i < dim_num; i++) {
274281
auto range_reader = ranges_reader[i];
275282
Datatype type = Datatype::UINT8;
276-
RETURN_NOT_OK(datatype_enum(range_reader.getType(), &type));
283+
throw_if_not_ok(datatype_enum(range_reader.getType(), &type));
284+
auto dim = array->array_schema_latest().dimension_ptr(i);
277285

278-
auto data = range_reader.getBuffer();
279-
auto data_ptr = data.asBytes();
286+
bool implicitly_initialized = range_reader.getHasDefaultRange();
287+
range_subset[i] =
288+
RangeSetAndSuperset(dim->type(), dim->domain(), true, coalesce_ranges);
289+
is_default[i] = implicitly_initialized;
280290
if (range_reader.hasBufferSizes()) {
281291
auto ranges = range_buffers_from_capnp(range_reader);
282-
RETURN_NOT_OK(subarray->set_ranges_for_dim(i, ranges));
283-
284-
// Set default indicator
285-
subarray->set_is_default(i, range_reader.getHasDefaultRange());
292+
// If the range is implicitly initialized, the RangeSetAndSuperset
293+
// constructor will initialize the ranges to the domain.
294+
if (!implicitly_initialized) {
295+
// Edge case for dimension labels where there are only label ranges set.
296+
if (ranges.empty()) {
297+
range_subset[i] = RangeSetAndSuperset(
298+
dim->type(), dim->domain(), false, coalesce_ranges);
299+
}
300+
// Add custom ranges, clearing any implicit ranges previously set.
301+
for (const auto& range : ranges) {
302+
throw_if_not_ok(range_subset[i].add_range_unrestricted(range));
303+
}
304+
}
286305
} else {
287306
// Handle 1.7 style ranges where there is a single range with no sizes
307+
auto data = range_reader.getBuffer();
308+
auto data_ptr = data.asBytes();
288309
Range range(data_ptr.begin(), data.size());
289-
RETURN_NOT_OK(subarray->set_ranges_for_dim(i, {range}));
290-
subarray->set_is_default(i, range_reader.getHasDefaultRange());
310+
throw_if_not_ok(range_subset[i].add_range_unrestricted(range));
291311
}
292312
}
293313

314+
std::vector<optional<Subarray::LabelRangeSubset>> label_range_subset(
315+
dim_num, nullopt);
294316
if (reader.hasLabelRanges()) {
295-
subarray->add_default_label_ranges(dim_num);
296317
auto label_ranges_reader = reader.getLabelRanges();
297318
uint32_t label_num = label_ranges_reader.size();
298319
for (uint32_t i = 0; i < label_num; i++) {
299320
auto label_range_reader = label_ranges_reader[i];
300-
auto dim_id = label_range_reader.getDimensionId();
321+
auto dim_index = label_range_reader.getDimensionId();
322+
auto dim = array->array_schema_latest().dimension_ptr(dim_index);
301323
auto label_name = label_range_reader.getName();
302324

303325
// Deserialize ranges for this dim label
304326
auto range_reader = label_range_reader.getRanges();
305327
auto ranges = range_buffers_from_capnp(range_reader);
306328

307329
// Set ranges for this dim label on the subarray
308-
subarray->set_label_ranges_for_dim(dim_id, label_name, ranges);
330+
label_range_subset[dim_index] = {
331+
label_name, dim->type(), coalesce_ranges};
309332
}
310333
}
311334

335+
std::unordered_map<std::string, std::vector<Range>> attr_range_subset;
312336
if (reader.hasAttributeRanges()) {
313-
std::unordered_map<std::string, std::vector<Range>> attr_ranges;
314337
auto attr_ranges_reader = reader.getAttributeRanges();
315338
if (attr_ranges_reader.hasEntries()) {
316-
for (auto attr_ranges_entry : attr_ranges_reader.getEntries()) {
317-
auto range_reader = attr_ranges_entry.getValue();
318-
auto key = std::string_view{
319-
attr_ranges_entry.getKey().cStr(),
320-
attr_ranges_entry.getKey().size()};
321-
attr_ranges[std::string{key}] = range_buffers_from_capnp(range_reader);
339+
for (auto entry : attr_ranges_reader.getEntries()) {
340+
auto range_reader = entry.getValue();
341+
std::string key{entry.getKey().cStr(), entry.getKey().size()};
342+
attr_range_subset[key] = range_buffers_from_capnp(range_reader);
322343
}
323344
}
324-
325-
for (const auto& attr_range : attr_ranges)
326-
subarray->set_attribute_ranges(attr_range.first, attr_range.second);
327345
}
328346

329347
// If cap'n proto object has stats set it on c++ object
348+
Subarray s(array, layout, parent_stats, logger, true);
330349
if (reader.hasStats()) {
331-
stats::Stats* stats = subarray->stats();
332-
// We should always have a stats here
333-
if (stats != nullptr) {
334-
RETURN_NOT_OK(stats_from_capnp(reader.getStats(), stats));
335-
}
350+
throw_if_not_ok(stats_from_capnp(reader.getStats(), s.stats()));
336351
}
337352

353+
std::vector<unsigned> relevant_fragments;
338354
if (reader.hasRelevantFragments()) {
339-
auto relevant_fragments = reader.getRelevantFragments();
340-
size_t count = relevant_fragments.size();
341-
std::vector<unsigned> rf;
342-
rf.reserve(count);
355+
auto reader_rf = reader.getRelevantFragments();
356+
size_t count = reader_rf.size();
357+
relevant_fragments.reserve(count);
343358
for (size_t i = 0; i < count; i++) {
344-
rf.emplace_back(relevant_fragments[i]);
359+
relevant_fragments.emplace_back(reader_rf[i]);
345360
}
346-
347-
subarray->relevant_fragments() = RelevantFragments(rf);
348361
}
349362

350-
return Status::Ok();
363+
return {
364+
array,
365+
layout,
366+
reader.hasStats() ? s.stats() : parent_stats,
367+
logger,
368+
range_subset,
369+
is_default,
370+
label_range_subset,
371+
attr_range_subset,
372+
relevant_fragments,
373+
coalesce_ranges};
351374
}
352375

353376
Status subarray_partitioner_to_capnp(
@@ -461,8 +484,8 @@ Status subarray_partitioner_from_capnp(
461484
RETURN_NOT_OK(layout_enum(subarray_reader.getLayout(), &layout));
462485

463486
// Subarray, which is used to initialize the partitioner.
464-
Subarray subarray(array, layout, query_stats, dummy_logger, true);
465-
RETURN_NOT_OK(subarray_from_capnp(reader.getSubarray(), &subarray));
487+
Subarray subarray = subarray_from_capnp(
488+
subarray_reader, array, layout, query_stats, dummy_logger);
466489
*partitioner = SubarrayPartitioner(
467490
&config,
468491
subarray,
@@ -519,10 +542,12 @@ Status subarray_partitioner_from_capnp(
519542
partition_info->end_ = partition_info_reader.getEnd();
520543
partition_info->split_multi_range_ =
521544
partition_info_reader.getSplitMultiRange();
522-
partition_info->partition_ =
523-
Subarray(array, layout, query_stats, dummy_logger, true);
524-
RETURN_NOT_OK(subarray_from_capnp(
525-
partition_info_reader.getSubarray(), &partition_info->partition_));
545+
partition_info->partition_ = subarray_from_capnp(
546+
partition_info_reader.getSubarray(),
547+
array,
548+
layout,
549+
query_stats,
550+
dummy_logger);
526551

527552
if (compute_current_tile_overlap) {
528553
throw_if_not_ok(partition_info->partition_.precompute_tile_overlap(
@@ -542,20 +567,18 @@ Status subarray_partitioner_from_capnp(
542567
auto sr_reader = state_reader.getSingleRange();
543568
const unsigned num_sr = sr_reader.size();
544569
for (unsigned i = 0; i < num_sr; i++) {
545-
auto subarray_reader_ = sr_reader[i];
546-
state->single_range_.emplace_back(
547-
array, layout, query_stats, dummy_logger, true);
548-
Subarray& subarray_ = state->single_range_.back();
549-
RETURN_NOT_OK(subarray_from_capnp(subarray_reader_, &subarray_));
570+
auto subarray_reader = sr_reader[i];
571+
Subarray subarray = subarray_from_capnp(
572+
subarray_reader, array, layout, query_stats, dummy_logger);
573+
state->single_range_.push_back(subarray);
550574
}
551575
auto m_reader = state_reader.getMultiRange();
552576
const unsigned num_m = m_reader.size();
553577
for (unsigned i = 0; i < num_m; i++) {
554-
auto subarray_reader_ = m_reader[i];
555-
state->multi_range_.emplace_back(
556-
array, layout, query_stats, dummy_logger, true);
557-
Subarray& subarray_ = state->multi_range_.back();
558-
RETURN_NOT_OK(subarray_from_capnp(subarray_reader_, &subarray_));
578+
auto subarray_reader = m_reader[i];
579+
Subarray subarray = subarray_from_capnp(
580+
subarray_reader, array, layout, query_stats, dummy_logger);
581+
state->multi_range_.push_back(subarray);
559582
}
560583

561584
// Overall mem budget
@@ -1131,9 +1154,9 @@ Status reader_from_capnp(
11311154
RETURN_NOT_OK(layout_enum(reader_reader.getLayout(), &layout));
11321155

11331156
// Subarray
1134-
Subarray subarray(array, layout, query->stats(), dummy_logger, true);
11351157
auto subarray_reader = reader_reader.getSubarray();
1136-
RETURN_NOT_OK(subarray_from_capnp(subarray_reader, &subarray));
1158+
Subarray subarray = subarray_from_capnp(
1159+
subarray_reader, array, layout, query->stats(), dummy_logger);
11371160
RETURN_NOT_OK(query->set_subarray_unsafe(subarray));
11381161

11391162
// Read state
@@ -1173,9 +1196,9 @@ Status index_reader_from_capnp(
11731196
RETURN_NOT_OK(layout_enum(reader_reader.getLayout(), &layout));
11741197

11751198
// Subarray
1176-
Subarray subarray(array, layout, query->stats(), dummy_logger, true);
11771199
auto subarray_reader = reader_reader.getSubarray();
1178-
RETURN_NOT_OK(subarray_from_capnp(subarray_reader, &subarray));
1200+
Subarray subarray = subarray_from_capnp(
1201+
subarray_reader, array, layout, query->stats(), dummy_logger);
11791202
RETURN_NOT_OK(query->set_subarray_unsafe(subarray));
11801203

11811204
// Read state
@@ -1216,9 +1239,9 @@ Status dense_reader_from_capnp(
12161239
RETURN_NOT_OK(layout_enum(reader_reader.getLayout(), &layout));
12171240

12181241
// Subarray
1219-
Subarray subarray(array, layout, query->stats(), dummy_logger, true);
12201242
auto subarray_reader = reader_reader.getSubarray();
1221-
RETURN_NOT_OK(subarray_from_capnp(subarray_reader, &subarray));
1243+
Subarray subarray = subarray_from_capnp(
1244+
subarray_reader, array, layout, query->stats(), dummy_logger);
12221245
RETURN_NOT_OK(query->set_subarray_unsafe(subarray));
12231246

12241247
// Read state
@@ -2257,9 +2280,9 @@ Status query_from_capnp(
22572280

22582281
// Subarray
22592282
if (writer_reader.hasSubarrayRanges()) {
2260-
Subarray subarray(array, layout, query->stats(), dummy_logger, true);
22612283
auto subarray_reader = writer_reader.getSubarrayRanges();
2262-
RETURN_NOT_OK(subarray_from_capnp(subarray_reader, &subarray));
2284+
Subarray subarray = subarray_from_capnp(
2285+
subarray_reader, array, layout, query->stats(), dummy_logger);
22632286
RETURN_NOT_OK(query->set_subarray_unsafe(subarray));
22642287
}
22652288
}
@@ -3207,9 +3230,9 @@ void ordered_dim_label_reader_from_capnp(
32073230
throw_if_not_ok(layout_enum(reader_reader.getLayout(), &layout));
32083231

32093232
// Subarray
3210-
Subarray subarray(array, layout, query->stats(), dummy_logger, false);
32113233
auto subarray_reader = reader_reader.getSubarray();
3212-
throw_if_not_ok(subarray_from_capnp(subarray_reader, &subarray));
3234+
Subarray subarray = subarray_from_capnp(
3235+
subarray_reader, array, layout, query->stats(), dummy_logger);
32133236
throw_if_not_ok(query->set_subarray_unsafe(subarray));
32143237

32153238
// OrderedDimLabelReader requires an initialized subarray for construction.

‎tiledb/sm/serialization/query.h‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -252,8 +252,12 @@ Status subarray_to_capnp(
252252
const Subarray* subarray,
253253
capnp::Subarray::Builder* builder);
254254

255-
Status subarray_from_capnp(
256-
const capnp::Subarray::Reader& reader, Subarray* subarray);
255+
Subarray subarray_from_capnp(
256+
const capnp::Subarray::Reader& reader,
257+
const Array* array,
258+
Layout layout,
259+
stats::Stats* parent_stats,
260+
shared_ptr<Logger> logger);
257261

258262
void ordered_dim_label_reader_to_capnp(
259263
const Query& query,

‎tiledb/sm/subarray/subarray.cc‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,32 @@ Subarray::Subarray(
150150
add_default_ranges();
151151
}
152152

153+
Subarray::Subarray(
154+
const Array* array,
155+
Layout layout,
156+
stats::Stats* stats,
157+
shared_ptr<Logger> logger,
158+
std::vector<RangeSetAndSuperset> range_subset,
159+
std::vector<bool> is_default,
160+
std::vector<optional<Subarray::LabelRangeSubset>> label_range_subset,
161+
std::unordered_map<std::string, std::vector<Range>> attr_range_subset,
162+
std::vector<unsigned int> relevant_fragments,
163+
bool coalesce_ranges)
164+
: stats_(stats)
165+
, logger_(std::move(logger))
166+
, array_(array->opened_array())
167+
, layout_(layout)
168+
, cell_order_(array_->array_schema_latest().cell_order())
169+
, range_subset_(std::move(range_subset))
170+
, label_range_subset_(std::move(label_range_subset))
171+
, attr_range_subset_(std::move(attr_range_subset))
172+
, is_default_(std::move(is_default))
173+
, est_result_size_computed_(false)
174+
, relevant_fragments_(relevant_fragments)
175+
, coalesce_ranges_(coalesce_ranges)
176+
, ranges_sorted_(false) {
177+
}
178+
153179
Subarray::Subarray(const Subarray& subarray)
154180
: Subarray() {
155181
// Make a deep-copy clone

0 commit comments

Comments
 (0)