Skip to content

Commit a89484c

Browse files
authored
Fix traversal limit in array deserialization (#4606)
This is a similar issue to the one fixed here: #4287, we are hitting the traversal limit when loading fragment metadata as part of opening the array when queryv3 is enabled. The fix is also similar, we just need to extend it to array deserialization, as we hit the same error when trying to open an array with large metadata. --- TYPE: IMPROVEMENT DESC: Fix traversal limit in array deserialization
1 parent b143210 commit a89484c

File tree

1 file changed

+16
-6
lines changed

1 file changed

+16
-6
lines changed

tiledb/sm/serialization/array.cc

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -299,9 +299,9 @@ Status array_from_capnp(
299299

300300
if (array_reader.hasFragmentMetadataAll()) {
301301
array->fragment_metadata().clear();
302-
array->fragment_metadata().reserve(
303-
array_reader.getFragmentMetadataAll().size());
304-
for (auto frag_meta_reader : array_reader.getFragmentMetadataAll()) {
302+
auto fragment_metadata_all_reader = array_reader.getFragmentMetadataAll();
303+
array->fragment_metadata().reserve(fragment_metadata_all_reader.size());
304+
for (auto frag_meta_reader : fragment_metadata_all_reader) {
305305
auto meta = make_shared<FragmentMetadata>(HERE());
306306
RETURN_NOT_OK(fragment_metadata_from_capnp(
307307
array->array_schema_latest_ptr(),
@@ -556,11 +556,21 @@ Status array_deserialize(
556556
break;
557557
}
558558
case SerializationType::CAPNP: {
559+
// Set traversal limit from config
560+
uint64_t limit = storage_manager->config()
561+
.get<uint64_t>("rest.capnp_traversal_limit")
562+
.value();
563+
::capnp::ReaderOptions readerOptions;
564+
// capnp uses the limit in words
565+
readerOptions.traversalLimitInWords = limit / sizeof(::capnp::word);
566+
559567
const auto mBytes =
560568
reinterpret_cast<const kj::byte*>(serialized_buffer.data());
561-
::capnp::FlatArrayMessageReader reader(kj::arrayPtr(
562-
reinterpret_cast<const ::capnp::word*>(mBytes),
563-
serialized_buffer.size() / sizeof(::capnp::word)));
569+
::capnp::FlatArrayMessageReader reader(
570+
kj::arrayPtr(
571+
reinterpret_cast<const ::capnp::word*>(mBytes),
572+
serialized_buffer.size() / sizeof(::capnp::word)),
573+
readerOptions);
564574
capnp::Array::Reader array_reader = reader.getRoot<capnp::Array>();
565575
RETURN_NOT_OK(array_from_capnp(array_reader, storage_manager, array));
566576
break;

0 commit comments

Comments
 (0)