Skip to content

Commit 3275bd5

Browse files
KiterLucKiterLuc
authored
Improve dense read performance for small reads. (#5145)
This improves a couples places that are easy gain when doing small dense reads. Running a small read in a loop (256K tile), I found out a few places where we can easily cut read times. 2 of those places were nested parallel fors that have been replaced by a parallel for 2d. The other was storing a full configuration class, which includes a map of ~148 configuration settings in the subarray. The subarray only uses two settings and those are now saved as member of the class, removing the need to generate a full configuration class when a subarray is constructed. [sc-50178] --- TYPE: IMPROVEMENT DESC: Improve dense read performance for small reads.
1 parent c11a7c6 commit 3275bd5

10 files changed

+100
-158
lines changed

‎test/src/unit-capi-sparse_array.cc‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3227,7 +3227,6 @@ TEST_CASE_METHOD(
32273227
tiledb_vfs_free(&vfs_);
32283228
// reallocate with input config
32293229
vfs_test_init(fs_vec_, &ctx_, &vfs_, config).ok();
3230-
tiledb_config_free(&config);
32313230

32323231
// Open array
32333232
tiledb_array_t* array;
@@ -3274,6 +3273,8 @@ TEST_CASE_METHOD(
32743273
REQUIRE(rc == TILEDB_OK);
32753274
rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s11[0], &s11[1], nullptr);
32763275
REQUIRE(rc == TILEDB_OK);
3276+
rc = tiledb_subarray_set_config(ctx_, subarray, config);
3277+
REQUIRE(rc == TILEDB_OK);
32773278
rc = tiledb_query_set_subarray_t(ctx_, query, subarray);
32783279
CHECK(rc == TILEDB_OK);
32793280

@@ -3297,6 +3298,7 @@ TEST_CASE_METHOD(
32973298
tiledb_query_free(&query);
32983299
tiledb_array_free(&array);
32993300
tiledb_subarray_free(&subarray);
3301+
tiledb_config_free(&config);
33003302
}
33013303

33023304
TEST_CASE_METHOD(

‎test/src/unit-cppapi-consolidation-with-timestamps.cc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1326,6 +1326,7 @@ TEST_CASE_METHOD(
13261326
Subarray subarray(ctx_, array);
13271327
subarray.add_range<uint64_t>(1, 2, 3);
13281328
subarray.add_range<uint64_t>(1, 2, 3);
1329+
subarray.set_config(cfg);
13291330
query.set_subarray(subarray);
13301331

13311332
// Submit/finalize the query

‎test/src/unit-cppapi-hilbert.cc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -388,6 +388,7 @@ TEST_CASE(
388388
subarray_r.add_range("d1", (int32_t)1, (int32_t)5);
389389
subarray_r.add_range("d2", (int32_t)1, (int32_t)3);
390390
subarray_r.add_range("d2", (int32_t)2, (int32_t)4);
391+
subarray_r.set_config(cfg);
391392
query_r.set_subarray(subarray_r);
392393
query_r.set_layout(TILEDB_UNORDERED);
393394
// Submit query

‎test/src/unit-cppapi-schema-evolution.cc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -478,8 +478,10 @@ TEST_CASE(
478478
// Prepare the query
479479
Query query(ctx, array, TILEDB_READ);
480480
Subarray subarray(ctx, array);
481+
subarray.set_config(cfg);
481482
subarray.add_range(0, 1, 4).add_range(0, 1, 4).add_range(1, 1, 4).add_range(
482483
1, 1, 4);
484+
subarray.set_config(cfg);
483485
query.set_subarray(subarray)
484486
.set_layout(layout)
485487
.set_data_buffer("a", a_data)

‎test/src/unit-cppapi-subarray.cc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1040,6 +1040,7 @@ TEST_CASE(
10401040
subarray.add_range(0, row_range[0], row_range[1]);
10411041
subarray.add_range(1, col_range0[0], col_range0[1]);
10421042
subarray.add_range(1, col_range1[0], col_range1[1]);
1043+
subarray.set_config(cfg);
10431044
query.set_subarray(subarray);
10441045
query.set_layout(TILEDB_UNORDERED);
10451046

@@ -1173,6 +1174,7 @@ TEST_CASE(
11731174
.set_data_buffer("a", data);
11741175

11751176
// Submit query
1177+
subarray.set_config(cfg);
11761178
query.set_subarray(subarray);
11771179
auto st = query.submit();
11781180
REQUIRE(st == tiledb::Query::Status::INCOMPLETE);

‎test/src/unit-sparse-unordered-with-dups-reader.cc‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ struct CSparseUnorderedWithDupsFx {
105105
uint8_t* a2_validity,
106106
uint64_t* a2_validity_size,
107107
uint64_t num_subarrays = 0,
108+
tiledb_config_t* config = nullptr,
108109
tiledb_query_t** query_ret = nullptr,
109110
tiledb_array_t** array_ret = nullptr);
110111
void reset_config();
@@ -520,6 +521,7 @@ int32_t CSparseUnorderedWithDupsFx::read_strings(
520521
uint8_t* a2_validity,
521522
uint64_t* a2_validity_size,
522523
uint64_t num_subarrays,
524+
tiledb_config_t* config,
523525
tiledb_query_t** query_ret,
524526
tiledb_array_t** array_ret) {
525527
// Open array for reading.
@@ -537,6 +539,10 @@ int32_t CSparseUnorderedWithDupsFx::read_strings(
537539
tiledb_subarray_t* subarray;
538540
rc = tiledb_subarray_alloc(ctx_, array, &subarray);
539541
CHECK(rc == TILEDB_OK);
542+
if (config != nullptr) {
543+
rc = tiledb_subarray_set_config(ctx_, subarray, config);
544+
CHECK(rc == TILEDB_OK);
545+
}
540546

541547
for (uint64_t i = 0; i < num_subarrays; i++) {
542548
// Create subarray for reading data.
@@ -1974,7 +1980,6 @@ TEST_CASE_METHOD(
19741980
REQUIRE(tiledb_ctx_alloc(config, &ctx_) == TILEDB_OK);
19751981
REQUIRE(error == nullptr);
19761982
REQUIRE(tiledb_vfs_alloc(ctx_, config, &vfs_) == TILEDB_OK);
1977-
tiledb_config_free(&config);
19781983

19791984
// Try to read with every possible buffer sizes. When varying
19801985
// buffer, the minimum should fit the number of dups at a minimum.
@@ -2013,6 +2018,7 @@ TEST_CASE_METHOD(
20132018
a2_validity_r.data(),
20142019
&a2_validity_r_size,
20152020
num_dups,
2021+
config,
20162022
&query,
20172023
&array);
20182024
CHECK(rc == TILEDB_OK);
@@ -2058,6 +2064,7 @@ TEST_CASE_METHOD(
20582064
tiledb_array_free(&array);
20592065
tiledb_query_free(&query);
20602066
}
2067+
tiledb_config_free(&config);
20612068
}
20622069

20632070
TEST_CASE_METHOD(
@@ -2123,7 +2130,6 @@ TEST_CASE_METHOD(
21232130
REQUIRE(tiledb_ctx_alloc(config, &ctx_) == TILEDB_OK);
21242131
REQUIRE(error == nullptr);
21252132
REQUIRE(tiledb_vfs_alloc(ctx_, config, &vfs_) == TILEDB_OK);
2126-
tiledb_config_free(&config);
21272133

21282134
tiledb_array_t* array;
21292135
auto st = tiledb_array_alloc(ctx_, array_name_.c_str(), &array);
@@ -2146,6 +2152,8 @@ TEST_CASE_METHOD(
21462152
CHECK(st == TILEDB_OK);
21472153
st = tiledb_subarray_add_range(ctx_, subarray, 0, &start, &end_2, nullptr);
21482154
CHECK(st == TILEDB_OK);
2155+
st = tiledb_subarray_set_config(ctx_, subarray, config);
2156+
CHECK(st == TILEDB_OK);
21492157
st = tiledb_query_set_subarray_t(ctx_, query, subarray);
21502158
CHECK(st == TILEDB_OK);
21512159
st = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED);
@@ -2187,4 +2195,5 @@ TEST_CASE_METHOD(
21872195
tiledb_array_free(&array);
21882196
tiledb_query_free(&query);
21892197
tiledb_subarray_free(&subarray);
2198+
tiledb_config_free(&config);
21902199
}

‎tiledb/sm/subarray/relevant_fragment_generator.cc‎

Lines changed: 35 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -98,58 +98,47 @@ RelevantFragments RelevantFragmentGenerator::compute_relevant_fragments(
9898
auto timer_se = stats_->start_timer("compute_relevant_frags");
9999
auto dim_num = array_->array_schema_latest().dim_num();
100100
auto fragment_num = array_->fragment_metadata().size();
101+
const auto meta = array_->fragment_metadata();
101102

102103
// Populate the fragment bytemap for each dimension in parallel.
103-
throw_if_not_ok(parallel_for(compute_tp, 0, dim_num, [&](const uint32_t d) {
104-
if (subarray_.is_default(d))
105-
return Status::Ok();
106-
107-
return compute_relevant_fragments_for_dim(
108-
compute_tp,
109-
d,
110-
fragment_num,
111-
start_coords_,
112-
end_coords_,
113-
&fragment_bytemaps_[d]);
114-
}));
104+
throw_if_not_ok(parallel_for_2d(
105+
compute_tp,
106+
0,
107+
dim_num,
108+
0,
109+
fragment_num,
110+
[&](const uint32_t d, const uint64_t f) {
111+
if (subarray_.is_default(d)) {
112+
return Status::Ok();
113+
}
114+
115+
// We're done when we have already determined fragment `f` to
116+
// be relevant for this dimension.
117+
if (fragment_bytemaps_[d][f] == 1) {
118+
return Status::Ok();
119+
}
120+
121+
auto dim{array_->array_schema_latest().dimension_ptr(d)};
122+
123+
// The fragment `f` is relevant to this dimension's fragment bytemap
124+
// if it overlaps with any range between the start and end coordinates
125+
// on this dimension.
126+
const type::Range& frag_range = meta[f]->non_empty_domain()[d];
127+
for (uint64_t r = start_coords_[d]; r <= end_coords_[d]; ++r) {
128+
const type::Range& query_range = subarray_.ranges_for_dim(d)[r];
129+
130+
if (dim->overlap(frag_range, query_range)) {
131+
fragment_bytemaps_[d][f] = 1;
132+
break;
133+
}
134+
}
135+
136+
return Status::Ok();
137+
}));
115138

116139
// Recalculate relevant fragments.
117140
return RelevantFragments(dim_num, fragment_num, fragment_bytemaps_);
118141
}
119142

120-
Status RelevantFragmentGenerator::compute_relevant_fragments_for_dim(
121-
ThreadPool* const compute_tp,
122-
const dimension_size_type dim_idx,
123-
const size_t fragment_num,
124-
const std::vector<uint64_t>& start_coords,
125-
const std::vector<uint64_t>& end_coords,
126-
std::vector<uint8_t>* const frag_bytemap) const {
127-
const auto meta = array_->fragment_metadata();
128-
auto dim{array_->array_schema_latest().dimension_ptr(dim_idx)};
129-
130-
return parallel_for(compute_tp, 0, fragment_num, [&](const uint64_t f) {
131-
// We're done when we have already determined fragment `f` to
132-
// be relevant for this dimension.
133-
if ((*frag_bytemap)[f] == 1) {
134-
return Status::Ok();
135-
}
136-
137-
// The fragment `f` is relevant to this dimension's fragment bytemap
138-
// if it overlaps with any range between the start and end coordinates
139-
// on this dimension.
140-
const type::Range& frag_range = meta[f]->non_empty_domain()[dim_idx];
141-
for (uint64_t r = start_coords[dim_idx]; r <= end_coords[dim_idx]; ++r) {
142-
const type::Range& query_range = subarray_.ranges_for_dim(dim_idx)[r];
143-
144-
if (dim->overlap(frag_range, query_range)) {
145-
(*frag_bytemap)[f] = 1;
146-
break;
147-
}
148-
}
149-
150-
return Status::Ok();
151-
});
152-
}
153-
154143
} // namespace sm
155144
} // namespace tiledb

‎tiledb/sm/subarray/relevant_fragment_generator.h‎

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -97,28 +97,6 @@ class RelevantFragmentGenerator {
9797
RelevantFragments compute_relevant_fragments(ThreadPool* compute_tp);
9898

9999
private:
100-
/* ********************************* */
101-
/* PRIVATE METHODS */
102-
/* ********************************* */
103-
104-
/**
105-
* Computes the relevant fragment bytemap for a specific dimension.
106-
*
107-
* @param compute_tp The thread pool for compute-bound tasks.
108-
* @param dim_idx The index of the dimension to compute on.
109-
* @param fragment_num The number of fragments to compute on.
110-
* @param start_coords The starting range coordinates to compute between.
111-
* @param end_coords The ending range coordinates to compute between.
112-
* @param frag_bytemap The fragment bytemap to mutate.
113-
*/
114-
Status compute_relevant_fragments_for_dim(
115-
ThreadPool* compute_tp,
116-
dimension_size_type dim_idx,
117-
size_t fragment_num,
118-
const std::vector<uint64_t>& start_coords,
119-
const std::vector<uint64_t>& end_coords,
120-
std::vector<uint8_t>* frag_bytemap) const;
121-
122100
/* ********************************* */
123101
/* PRIVATE ATTRIBUTES */
124102
/* ********************************* */

0 commit comments

Comments
 (0)