Merge pull request ClickHouse#86941 from ClickHouse/backport/25.8/86812

Backport ClickHouse#86812 to 25.8: Add addition validations in ColumnObject

Author: Avogar

src/Columns/ColumnObject.cpp

@@ -631,6 +631,7 @@ void ColumnObject::doInsertFrom(const IColumn & src, size_t n)
 
     /// Finally, insert paths from shared data.
     insertFromSharedDataAndFillRemainingDynamicPaths(src_object_column, std::move(src_dynamic_paths_for_shared_data), n, 1);
+    validateDynamicPathsSizes();
 }
 
 #if !defined(DEBUG_OR_SANITIZER_BUILD)
@@ -665,6 +666,7 @@ void ColumnObject::doInsertRangeFrom(const IColumn & src, size_t start, size_t l
 
     /// Finally, insert paths from shared data.
     insertFromSharedDataAndFillRemainingDynamicPaths(src_object_column, std::move(src_dynamic_paths_for_shared_data), start, length);
+    validateDynamicPathsSizes();
 }
 
 void ColumnObject::insertFromSharedDataAndFillRemainingDynamicPaths(const DB::ColumnObject & src_object_column, std::vector<std::string_view> && src_dynamic_paths_for_shared_data, size_t start, size_t length)
@@ -726,6 +728,8 @@ void ColumnObject::insertFromSharedDataAndFillRemainingDynamicPaths(const DB::Co
             if (auto it = dynamic_paths_ptrs.find(path); it != dynamic_paths_ptrs.end())
             {
                 /// Deserialize binary value into dynamic column from shared data.
+                if (it->second->size() != current_size)
+                    throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected size of dynamic path {}: {} != {}. It may indicate duplicated data for this path", path, it->second->size(), current_size);
                 deserializeValueFromSharedData(src_shared_data_values, i, *it->second);
             }
             else
@@ -1995,4 +1999,14 @@ int ColumnObject::doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_d
     return 1;
 }
 
+void ColumnObject::validateDynamicPathsSizes() const
+{
+    size_t expected_size = shared_data->size();
+    for (const auto & [path, column] : dynamic_paths)
+    {
+        if (column->size() != expected_size)
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected size of dynamic path {}: {} != {}", path, column->size(), expected_size);
+    }
+}
+
 }

src/Columns/ColumnObject.h

@@ -279,6 +279,9 @@ class ColumnObject final : public COWHelper<IColumnHelper<ColumnObject>, ColumnO
     /// Insert all the data from shared data with specified path to dynamic column.
     static void fillPathColumnFromSharedData(IColumn & path_column, StringRef path, const ColumnPtr & shared_data_column, size_t start, size_t end);
 
+    /// Validate that all dynamic paths have correct sizes.
+    void validateDynamicPathsSizes() const;
+
 private:
     class SortedPathsIterator;
 

src/DataTypes/Serializations/SerializationObject.cpp

@@ -780,6 +780,7 @@ void SerializationObject::serializeBinaryBulkWithMultipleStreams(
     }
 
     const auto & column_object = assert_cast<const ColumnObject &>(column);
+    column_object.validateDynamicPathsSizes();
     const auto & typed_paths = column_object.getTypedPaths();
 
     if (object_state->serialization_version.value == SerializationVersion::FLATTENED)
@@ -1035,6 +1036,8 @@ void SerializationObject::deserializeBinaryBulkWithMultipleStreams(
     object_state->shared_data_serialization->deserializeBinaryBulkWithMultipleStreams(shared_data, rows_offset, limit, settings, object_state->shared_data_state, cache);
     settings.path.pop_back();
     settings.path.pop_back();
+
+    column_object.validateDynamicPathsSizes();
 }
 
 void SerializationObject::serializeBinary(const Field & field, WriteBuffer & ostr, const DB::FormatSettings & settings) const
@@ -1226,7 +1229,9 @@ void SerializationObject::deserializeBinary(IColumn & col, ReadBuffer & istr, co
                     String value;
                     Field field;
                     readParsedValueIntoString(value, istr, [&](ReadBuffer & buf){ dynamic_serialization->deserializeBinary(field, buf, settings); });
-                    paths_and_values_for_shared_data.emplace_back(std::move(path), std::move(value));
+                    /// Don't write nulls into shared data.
+                    if (!field.isNull())
+                        paths_and_values_for_shared_data.emplace_back(std::move(path), std::move(value));
                 }
             }
             else
@@ -1272,6 +1277,8 @@ void SerializationObject::deserializeBinary(IColumn & col, ReadBuffer & istr, co
         if (column->size() == prev_size)
             column->insertDefault();
     }
+
+    column_object.validateDynamicPathsSizes();
 }
 
 SerializationPtr SerializationObject::TypedPathSubcolumnCreator::create(const DB::SerializationPtr & prev, const DataTypePtr &) const

tests/queries/0_stateless/03621_json_null_in_shared_data.reference

@@ -0,0 +1,2 @@
+00000000  00                                                |.|
+00000001

tests/queries/0_stateless/03621_json_null_in_shared_data.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+$CLICKHOUSE_LOCAL -q "select '{\"a\" : null}'::JSON(a Dynamic) format RowBinary" | $CLICKHOUSE_LOCAL -q "select * from table format RowBinary" --input-format RowBinary --structure "json JSON(max_dynamic_paths=0)" | hexdump -C