[check](column) Add nullable checks for array/map.

be/src/vec/columns/column.cpp

@@ -21,8 +21,11 @@
 #include "vec/columns/column.h"
 
 #include "util/simd/bits.h"
+#include "vec/columns/column_array.h"
 #include "vec/columns/column_const.h"
+#include "vec/columns/column_map.h"
 #include "vec/columns/column_nullable.h"
+#include "vec/columns/column_variant.h"
 #include "vec/core/sort_block.h"
 #include "vec/data_types/data_type.h"
 
@@ -89,6 +92,39 @@ bool IColumn::null_map_check() const {
     return is_valid;
 }
 
+bool IColumn::complex_type_data_nullable_check() const {
+    if (check_and_get_column<ColumnVariant>(*this)) {
+        // skip check for ColumnVariant
+        return true;
+    }
+
+    auto check_complex_data_is_nullable = [&](const IColumn& subcolumn) {
+        if (const auto* col_arr = check_and_get_column<ColumnArray>(subcolumn)) {
+            const auto& data_col = col_arr->get_data();
+            if (!is_column_nullable(data_col)) {
+                return false;
+            }
+        } else if (const auto* col_map = check_and_get_column<ColumnMap>(subcolumn)) {
+            const auto& keys_col = col_map->get_keys();
+            const auto& values_col = col_map->get_values();
+            if (!is_column_nullable(keys_col) || !is_column_nullable(values_col)) {
+                return false;
+            }
+        }
+        return true;
+    };
+
+    bool is_valid = check_complex_data_is_nullable(*this);
+    ColumnCallback callback = [&](ColumnPtr& subcolumn) {
+        if (!subcolumn->complex_type_data_nullable_check()) {
+            is_valid = false;
+        }
+    };
+    // simply read using for_each_subcolumn without modification; const_cast can be used.
+    const_cast<IColumn*>(this)->for_each_subcolumn(callback);
+    return is_valid;
+}
+
 Status IColumn::column_self_check() const {
 #ifndef NDEBUG
     // check const nested
@@ -100,6 +136,11 @@ Status IColumn::column_self_check() const {
     if (!null_map_check()) {
         return Status::InternalError("null map check failed for column: {}", get_name());
     }
+    // check complex type data nullable
+    if (!complex_type_data_nullable_check()) {
+        return Status::InternalError("complex type data nullable check failed for column: {} ",
+                                     dump_structure());
+    }
 #endif
     return Status::OK();
 }

be/src/vec/columns/column.h

@@ -666,6 +666,8 @@ class IColumn : public COW<IColumn> {
 
     bool null_map_check() const;
 
+    bool complex_type_data_nullable_check() const;
+
     // const column nested check, eg. const(nullable(...)) is allowed
     //  const(array(const(...))) is not allowed
     bool const_nested_check() const;

be/src/vec/core/block.cpp

@@ -279,6 +279,9 @@ Status Block::check_type_and_column() const {
         if (!elem.type) {
             continue;
         }
+        if (elem.name.find(BeConsts::SPARSE_COLUMN_PATH) != std::string::npos) {
+            continue;
+        }
 
         // ColumnNothing is a special column type, it is used to represent a column that
         // is not materialized, so we don't need to check it.

be/test/vec/columns/column_self_check.cpp

@@ -22,6 +22,7 @@
 #include "vec/columns/column.h"
 #include "vec/columns/column_array.h"
 #include "vec/columns/column_const.h"
+#include "vec/columns/column_map.h"
 #include "vec/data_types/data_type_number.h"
 #include "vec/functions/function.h"
 #include "vec/functions/simple_function_factory.h"
@@ -119,5 +120,35 @@ TEST(ColumnSelfCheckTest, nullmap_check_test) {
         EXPECT_EQ(col_const->column_self_check(), false);
     }
 }
+TEST(ColumnSelfCheckTest, complex_type_data_nullable_check) {
+    // array/map data column is nullable check
+    {
+        auto data_col = ColumnHelper::create_nullable_column<DataTypeInt32>({1, 2, 3}, {0, 0, 0});
+        auto offsets_col = ColumnArray::ColumnOffsets::create(0, 3);
+        auto array_col = ColumnArray::create(data_col, std::move(offsets_col));
+
+        EXPECT_EQ(array_col->complex_type_data_nullable_check(), true);
+    }
+    {
+        auto data_col = ColumnHelper::create_column<DataTypeInt32>({1, 2, 3});
+        auto offsets_col = ColumnArray::ColumnOffsets::create(0, 3);
+        auto array_col = ColumnArray::create(data_col, std::move(offsets_col));
+        EXPECT_EQ(array_col->complex_type_data_nullable_check(), false);
+    }
+    {
+        auto keys_col = ColumnHelper::create_nullable_column<DataTypeInt32>({1, 2, 3}, {0, 0, 0});
+        auto values_col = ColumnHelper::create_nullable_column<DataTypeInt32>({4, 5, 6}, {0, 0, 0});
+        auto offsets_col = ColumnArray::ColumnOffsets::create(0, 3);
+        auto map_col = ColumnMap::create(keys_col, values_col, std::move(offsets_col));
+        EXPECT_EQ(map_col->complex_type_data_nullable_check(), true);
+    }
+    {
+        auto keys_col = ColumnHelper::create_column<DataTypeInt32>({1, 2, 3});
+        auto values_col = ColumnHelper::create_nullable_column<DataTypeInt32>({4, 5, 6}, {0, 0, 0});
+        auto offsets_col = ColumnArray::ColumnOffsets::create(0, 3);
+        auto map_col = ColumnMap::create(keys_col, values_col, std::move(offsets_col));
+        EXPECT_EQ(map_col->complex_type_data_nullable_check(), false);
+    }
+}
 
 } // namespace doris::vectorized