Skip to content

Commit eb17b2e

Browse files
authored
[Fix](multi-catalog) Fix some undefined behaviors. (apache#37845)
## Proposed changes - Null pointer of type 'doris::StringRef' in orc reader. The root cause is error will throw when `num_values == 0` in `_decode_string_non_dict_encoded_column`. ``` /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/stl_vector.h:1046:9: runtime error: reference binding to null pointer of type 'doris::StringRef' #0 0x562516fa9770 in std::vector<doris::StringRef, std::allocator<doris::StringRef> >::operator[](unsigned long) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/stl_vector.h:1046:2 #1 0x562516fa9770 in doris::Status doris::vectorized::OrcReader::_decode_string_non_dict_encoded_column<false>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, COW<doris::vectorized::IColumn>::mutable_ptr<doris::vectorized::IColumn> const&, orc::TypeKind const&, orc::EncodedStringVectorBatch*, unsigned long) /root/doris/be/src/vec/exec/format/orc/vorc_reader.cpp:1164:39 #2 0x562516f9c08b in doris::Status doris::vectorized::OrcReader::_decode_string_column<false>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, COW<doris::vectorized::IColumn>::mutable_ptr<doris::vectorized::IColumn> const&, orc::TypeKind const&, orc::ColumnVectorBatch*, unsigned long) /root/doris/be/src/vec/exec/format/orc/vorc_reader.cpp:1116:16 #3 0x562516f91d73 in doris::Status doris::vectorized::OrcReader::_fill_doris_data_column<false>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, COW<doris::vectorized::IColumn>::mutable_ptr<doris::vectorized::IColumn>&, std::shared_ptr<doris::vectorized::IDataType const> const&, orc::Type const*, orc::ColumnVectorBatch*, unsigned long) /root/doris/be/src/vec/exec/format/orc/vorc_reader.cpp:1357:16 #4 0x562516c79a0c in doris::Status doris::vectorized::OrcReader::_orc_column_to_doris_column<false>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, COW<doris::vectorized::IColumn>::immutable_ptr<doris::vectorized::IColumn>&, std::shared_ptr<doris::vectorized::IDataType const> const&, orc::Type const*, orc::ColumnVectorBatch*, unsigned long) /root/doris/be/src/vec/exec/format/orc/vorc_reader.cpp:1524:5 #5 0x562516f9339a in doris::Status doris::vectorized::OrcReader::_fill_doris_data_column<false>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, COW<doris::vectorized::IColumn>::mutable_ptr<doris::vectorized::IColumn>&, std::shared_ptr<doris::vectorized::IDataType const> const&, orc::Type const*, orc::ColumnVectorBatch*, unsigned long) /root/doris/be/src/vec/exec/format/orc/vorc_reader.cpp:1402:9 #6 0x562516c79a0c in doris::Status doris::vectorized::OrcReader::_orc_column_to_doris_column<false>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, COW<doris::vectorized::IColumn>::immutable_ptr<doris::vectorized::IColumn>&, std::shared_ptr<doris::vectorized::IDataType const> const&, orc::Type const*, orc::ColumnVectorBatch*, unsigned long) /root/doris/be/src/vec/exec/format/orc/vorc_reader.cpp:1524:5 ... ``` - Shift exponent 128 is too large for 128-bit type 'ValueCopyType' (aka '__int128') in parquet reader. The root cause is error will throw when `len == 0`. ``` /root/doris/be/src/vec/exec/format/parquet/parquet_column_convert.h:413:27: runtime error: shift exponent 128 is too large for 128-bit type 'ValueCopyType' (aka '__int128') #0 0x56251760fbc7 in doris::vectorized::parquet::StringToDecimal<doris::vectorized::Decimal128V3, (doris::vectorized::DecimalScaleParams::ScaleType)1>::physical_convert(COW<doris::vectorized::IColumn>::immutable_ptr<doris::vectorized::IColumn>&, COW<doris::vectorized::IColumn>::immutable_ptr<doris::vectorized::IColumn>&) /root/doris/be/src/vec/exec/format/parquet/parquet_column_convert.h:413:27 #1 0x562517290dc4 in doris::vectorized::parquet::PhysicalToLogicalConverter::convert(COW<doris::vectorized::IColumn>::immutable_ptr<doris::vectorized::IColumn>&, doris::TypeDescriptor, std::shared_ptr<doris::vectorized::IDataType const> const&, COW<doris::vectorized::IColumn>::immutable_ptr<doris::vectorized::IColumn>&, bool) /root/doris/be/src/vec/exec/format/parquet/parquet_column_convert.h:209:9 #2 0x562517284a6d in doris::vectorized::ScalarColumnReader::read_column_data(COW<doris::vectorized::IColumn>::immutable_ptr<doris::vectorized::IColumn>&, std::shared_ptr<doris::vectorized::IDataType const>&, doris::vectorized::ColumnSelectVector&, unsigned long, unsigned long*, bool*, bool) /root/doris/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp:569:24 #3 0x56251725ae7e in doris::vectorized::RowGroupReader::_read_column_data(doris::vectorized::Block*, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, unsigned long, unsigned long*, bool*, doris::vectorized::ColumnSelectVector&) /root/doris/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp:421:13 #4 0x56251724d6d2 in doris::vectorized::RowGroupReader::next_batch(doris::vectorized::Block*, unsigned long, unsigned long*, bool*) /root/doris/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp:321:9 #5 0x56251708eb97 in doris::vectorized::ParquetReader::get_next_block(doris::vectorized::Block*, unsigned long*, bool*) /root/doris/be/src/vec/exec/format/parquet/vparquet_reader.cpp:530:36 #6 0x56253036772d in doris::vectorized::VFileScanner::_get_block_wrapped(doris::RuntimeState*, doris::vectorized::Block*, bool*) /root/doris/be/src/vec/exec/scan/vfile_scanner.cpp:311:13 #7 0x562530366549 in doris::vectorized::VFileScanner::_get_block_impl(doris::RuntimeState*, doris::vectorized::Block*, bool*) /root/doris/be/src/vec/exec/scan/vfile_scanner.cpp:253:17 #8 0x5625176e79c8 in doris::vectorized::VScanner::get_block(doris::RuntimeState*, doris::vectorized::Block*, bool*) /root/doris/be/src/vec/exec/scan/vscanner.cpp:117:17 #9 0x5625176e6fc1 in doris::vectorized::VScanner::get_block_after_projects(doris::RuntimeState*, doris::vectorized::Block*, bool*) /root/doris/be/src/vec/exec/scan/vscanner.cpp:84:12 #10 0x562517698047 in doris::vectorized::ScannerScheduler::_scanner_scan(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>) /root/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:250:5 #11 0x56251769bc1f in doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()::operator()() const::'lambda'()::operator()() const /root/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:172:25 #12 0x56251769bc1f in doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()::operator()() const /root/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:171:35 ... ```
1 parent a1b0264 commit eb17b2e

File tree

2 files changed

+16
-14
lines changed

2 files changed

+16
-14
lines changed

be/src/vec/exec/format/orc/vorc_reader.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1583,7 +1583,7 @@ Status OrcReader::get_next_block_impl(Block* block, size_t* read_rows, bool* eof
15831583
_decimal_scale_params_index = 0;
15841584
try {
15851585
rr = _row_reader->nextBatch(*_batch, block);
1586-
if (rr == 0) {
1586+
if (rr == 0 || _batch->numElements == 0) {
15871587
*eof = true;
15881588
*read_rows = 0;
15891589
return Status::OK();
@@ -1653,7 +1653,7 @@ Status OrcReader::get_next_block_impl(Block* block, size_t* read_rows, bool* eof
16531653
_decimal_scale_params_index = 0;
16541654
try {
16551655
rr = _row_reader->nextBatch(*_batch, block);
1656-
if (rr == 0) {
1656+
if (rr == 0 || _batch->numElements == 0) {
16571657
*eof = true;
16581658
*read_rows = 0;
16591659
return Status::OK();

be/src/vec/exec/format/parquet/parquet_column_convert.h

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -408,18 +408,20 @@ class StringToDecimal : public PhysicalToLogicalConverter {
408408
// When Decimal in parquet is stored in byte arrays, binary and fixed,
409409
// the unscaled number must be encoded as two's complement using big-endian byte order.
410410
ValueCopyType value = 0;
411-
memcpy(reinterpret_cast<char*>(&value), buf + offset[i - 1], len);
412-
value = BitUtil::big_endian_to_host(value);
413-
value = value >> ((sizeof(value) - len) * 8);
414-
if constexpr (ScaleType == DecimalScaleParams::SCALE_UP) {
415-
value *= scale_params.scale_factor;
416-
} else if constexpr (ScaleType == DecimalScaleParams::SCALE_DOWN) {
417-
value /= scale_params.scale_factor;
418-
} else if constexpr (ScaleType == DecimalScaleParams::NO_SCALE) {
419-
// do nothing
420-
} else {
421-
LOG(FATAL) << "__builtin_unreachable";
422-
__builtin_unreachable();
411+
if (len > 0) {
412+
memcpy(reinterpret_cast<char*>(&value), buf + offset[i - 1], len);
413+
value = BitUtil::big_endian_to_host(value);
414+
value = value >> ((sizeof(value) - len) * 8);
415+
if constexpr (ScaleType == DecimalScaleParams::SCALE_UP) {
416+
value *= scale_params.scale_factor;
417+
} else if constexpr (ScaleType == DecimalScaleParams::SCALE_DOWN) {
418+
value /= scale_params.scale_factor;
419+
} else if constexpr (ScaleType == DecimalScaleParams::NO_SCALE) {
420+
// do nothing
421+
} else {
422+
LOG(FATAL) << "__builtin_unreachable";
423+
__builtin_unreachable();
424+
}
423425
}
424426
auto& v = reinterpret_cast<DecimalType&>(data[start_idx + i]);
425427
v = (DecimalType)value;

0 commit comments

Comments
 (0)