diff --git a/be/src/vec/columns/column.cpp b/be/src/vec/columns/column.cpp index f6f595fc6cdc47..9a750e9bca278e 100644 --- a/be/src/vec/columns/column.cpp +++ b/be/src/vec/columns/column.cpp @@ -21,8 +21,11 @@ #include "vec/columns/column.h" #include "util/simd/bits.h" +#include "vec/columns/column_array.h" #include "vec/columns/column_const.h" +#include "vec/columns/column_map.h" #include "vec/columns/column_nullable.h" +#include "vec/columns/column_variant.h" #include "vec/core/sort_block.h" #include "vec/data_types/data_type.h" @@ -89,6 +92,39 @@ bool IColumn::null_map_check() const { return is_valid; } +bool IColumn::complex_type_data_nullable_check() const { + if (check_and_get_column(*this)) { + // skip check for ColumnVariant + return true; + } + + auto check_complex_data_is_nullable = [&](const IColumn& subcolumn) { + if (const auto* col_arr = check_and_get_column(subcolumn)) { + const auto& data_col = col_arr->get_data(); + if (!is_column_nullable(data_col)) { + return false; + } + } else if (const auto* col_map = check_and_get_column(subcolumn)) { + const auto& keys_col = col_map->get_keys(); + const auto& values_col = col_map->get_values(); + if (!is_column_nullable(keys_col) || !is_column_nullable(values_col)) { + return false; + } + } + return true; + }; + + bool is_valid = check_complex_data_is_nullable(*this); + ColumnCallback callback = [&](ColumnPtr& subcolumn) { + if (!subcolumn->complex_type_data_nullable_check()) { + is_valid = false; + } + }; + // simply read using for_each_subcolumn without modification; const_cast can be used. + const_cast(this)->for_each_subcolumn(callback); + return is_valid; +} + Status IColumn::column_self_check() const { #ifndef NDEBUG // check const nested @@ -100,6 +136,11 @@ Status IColumn::column_self_check() const { if (!null_map_check()) { return Status::InternalError("null map check failed for column: {}", get_name()); } + // check complex type data nullable + if (!complex_type_data_nullable_check()) { + return Status::InternalError("complex type data nullable check failed for column: {} ", + dump_structure()); + } #endif return Status::OK(); } diff --git a/be/src/vec/columns/column.h b/be/src/vec/columns/column.h index 1fbbdae8992ca0..3249e6bdb7b268 100644 --- a/be/src/vec/columns/column.h +++ b/be/src/vec/columns/column.h @@ -666,6 +666,8 @@ class IColumn : public COW { bool null_map_check() const; + bool complex_type_data_nullable_check() const; + // const column nested check, eg. const(nullable(...)) is allowed // const(array(const(...))) is not allowed bool const_nested_check() const; diff --git a/be/src/vec/core/block.cpp b/be/src/vec/core/block.cpp index 33504640b60d5d..9fa5010c62b61e 100644 --- a/be/src/vec/core/block.cpp +++ b/be/src/vec/core/block.cpp @@ -279,6 +279,9 @@ Status Block::check_type_and_column() const { if (!elem.type) { continue; } + if (elem.name.find(BeConsts::SPARSE_COLUMN_PATH) != std::string::npos) { + continue; + } // ColumnNothing is a special column type, it is used to represent a column that // is not materialized, so we don't need to check it. diff --git a/be/test/vec/columns/column_self_check.cpp b/be/test/vec/columns/column_self_check.cpp index 13fcfcbf5f0907..d48bdfb48af6fc 100644 --- a/be/test/vec/columns/column_self_check.cpp +++ b/be/test/vec/columns/column_self_check.cpp @@ -22,6 +22,7 @@ #include "vec/columns/column.h" #include "vec/columns/column_array.h" #include "vec/columns/column_const.h" +#include "vec/columns/column_map.h" #include "vec/data_types/data_type_number.h" #include "vec/functions/function.h" #include "vec/functions/simple_function_factory.h" @@ -119,5 +120,35 @@ TEST(ColumnSelfCheckTest, nullmap_check_test) { EXPECT_EQ(col_const->column_self_check(), false); } } +TEST(ColumnSelfCheckTest, complex_type_data_nullable_check) { + // array/map data column is nullable check + { + auto data_col = ColumnHelper::create_nullable_column({1, 2, 3}, {0, 0, 0}); + auto offsets_col = ColumnArray::ColumnOffsets::create(0, 3); + auto array_col = ColumnArray::create(data_col, std::move(offsets_col)); + + EXPECT_EQ(array_col->complex_type_data_nullable_check(), true); + } + { + auto data_col = ColumnHelper::create_column({1, 2, 3}); + auto offsets_col = ColumnArray::ColumnOffsets::create(0, 3); + auto array_col = ColumnArray::create(data_col, std::move(offsets_col)); + EXPECT_EQ(array_col->complex_type_data_nullable_check(), false); + } + { + auto keys_col = ColumnHelper::create_nullable_column({1, 2, 3}, {0, 0, 0}); + auto values_col = ColumnHelper::create_nullable_column({4, 5, 6}, {0, 0, 0}); + auto offsets_col = ColumnArray::ColumnOffsets::create(0, 3); + auto map_col = ColumnMap::create(keys_col, values_col, std::move(offsets_col)); + EXPECT_EQ(map_col->complex_type_data_nullable_check(), true); + } + { + auto keys_col = ColumnHelper::create_column({1, 2, 3}); + auto values_col = ColumnHelper::create_nullable_column({4, 5, 6}, {0, 0, 0}); + auto offsets_col = ColumnArray::ColumnOffsets::create(0, 3); + auto map_col = ColumnMap::create(keys_col, values_col, std::move(offsets_col)); + EXPECT_EQ(map_col->complex_type_data_nullable_check(), false); + } +} } // namespace doris::vectorized \ No newline at end of file