fix

Author: csun5285

be/src/olap/rowset/segment_v2/segment_iterator.cpp

@@ -2363,6 +2363,8 @@ Status SegmentIterator::next_batch(vectorized::Block* block) {
 
             RETURN_IF_ERROR(block->check_type_and_column());
 
+            LOG(INFO) << "block data: " << block->dump_data();
+
             return Status::OK();
         });
     }();

be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.cpp

@@ -1132,8 +1132,6 @@ Status VariantCompactionDocSnapshotWriter::finalize() {
             if (vectorized::schema_util::generate_sub_column_info(
                         *_opts.rowset_ctx->tablet_schema, parent_column.unique_id(),
                         std::string(path), &sub_column_info)) {
-                vectorized::schema_util::inherit_column_attributes(parent_column,
-                                                                   sub_column_info.column);
                 tablet_column = std::move(sub_column_info.column);
                 _subcolumns_indexes[column_id] = std::move(sub_column_info.indexes);
             } else {

be/src/olap/tablet_schema.h

@@ -245,7 +245,8 @@ class TabletColumn : public MetadataAdder<TabletColumn> {
         _variant_doc_snapshot_shard_count = variant_doc_snapshot_shard_count;
     }
 
-    void set_variant_max_sparse_column_statistics_size(int32_t variant_max_sparse_column_statistics_size) {
+    void set_variant_max_sparse_column_statistics_size(
+            int32_t variant_max_sparse_column_statistics_size) {
         _variant_max_sparse_column_statistics_size = variant_max_sparse_column_statistics_size;
     }
 

be/src/vec/columns/column_variant.cpp

@@ -727,6 +727,10 @@ void ColumnVariant::try_insert(const Field& field) {
     size_t old_size = size();
     const auto& object = field.get<const VariantMap&>();
     for (const auto& [key, value] : object) {
+        if (key.get_path() == "__DORIS_VARIANT_DOC_SNAPSHOT__") {
+            insert_to_doc_snapshot_column(value.field);
+            continue;
+        }
         if (!has_subcolumn(key)) {
             bool succ = add_sub_column(key, old_size);
             if (!succ) {
@@ -757,7 +761,9 @@ void ColumnVariant::try_insert(const Field& field) {
         }
     }
     serialized_sparse_column->insert_default();
-    serialized_doc_snapshot_column->insert_default();
+    if (serialized_doc_snapshot_column->size() == old_size) {
+        serialized_doc_snapshot_column->insert_default();
+    }
     ++num_rows;
     ENABLE_CHECK_CONSISTENCY(this);
 }
@@ -930,28 +936,33 @@ void ColumnVariant::get(size_t n, Field& res) const {
                                               .precision = data.second.precision,
                                               .scale = data.second.scale});
     }
-
-    const auto& [doc_snapshot_path, doc_snapshot_value] = get_doc_snapshot_data_paths_and_values();
-    const auto& doc_snapshot_column_offsets = serialized_doc_snapshot_column_offsets();
-    size_t doc_snapshot_offset = doc_snapshot_column_offsets[n - 1];
-    size_t doc_snapshot_end = doc_snapshot_column_offsets[n];
-    for (size_t i = doc_snapshot_offset; i != doc_snapshot_end; ++i) {
-        const StringRef path_data = doc_snapshot_path->get_data_at(i);
-        auto data = ColumnVariant::deserialize_from_binary_column(doc_snapshot_value, i);
-        object.try_emplace(PathInData(path_data),
-                           FieldWithDataType {.field = std::move(data.first),
-                                              .base_scalar_type_id = data.second.scalar_type_id,
-                                              .num_dimensions = static_cast<uint8_t>(
-                                                      data.second.num_dimensions),
-                                              .precision = data.second.precision,
-                                              .scale = data.second.scale});
-    }
-
+    try_get_from_doc_snapshot_column(n, res);
     if (object.empty()) {
         res = Field();
     }
 }
 
+bool ColumnVariant::has_doc_snapshot_column(size_t n) const {
+    const auto& offsets = serialized_doc_snapshot_column_offsets();
+    return offsets[n - 1] < offsets[n];
+}
+
+void ColumnVariant::try_get_from_doc_snapshot_column(size_t n, Field& res) const {
+    if (!has_doc_snapshot_column(n)) {
+        return;
+    }
+    FieldWithDataType field_with_data_type;
+    serialized_doc_snapshot_column->get(n, field_with_data_type.field);
+    auto& object = res.get<VariantMap&>();
+    object.try_emplace(PathInData("__DORIS_VARIANT_DOC_SNAPSHOT__"),
+                       std::move(field_with_data_type));
+}
+
+void ColumnVariant::insert_to_doc_snapshot_column(const Field& field) {
+    serialized_doc_snapshot_column->insert(field);
+    CHECK(subcolumns.size() == 1) << "subcolumns size should be 1";
+}
+
 void ColumnVariant::add_nested_subcolumn(const PathInData& key, const FieldInfo& field_info,
                                          size_t new_size) {
     if (!key.has_nested_part()) {

be/src/vec/columns/column_variant.h

@@ -633,6 +633,12 @@ class ColumnVariant final : public COWHelper<IColumn, ColumnVariant> {
     // doc snapshot mode: only root column, and doc snapshot column is not empty
     bool is_doc_snapshot_mode() const;
 
+    void try_get_from_doc_snapshot_column(size_t n, Field& res) const;
+
+    void insert_to_doc_snapshot_column(const Field& field);
+
+    bool has_doc_snapshot_column(size_t n) const;
+
 private:
     // May throw execption
     void try_insert(const Field& field);

be/src/vec/common/schema_util.cpp

@@ -609,8 +609,6 @@ TabletColumn create_doc_snapshot_column(const TabletColumn& variant, int bucket_
     res.set_parent_unique_id(variant.unique_id());
     res.set_default_value("NULL");
     res.set_path_info(PathInData {name});
-    res.set_variant_doc_snapshot_min_rows(variant.variant_doc_snapshot_min_rows());
-    res.set_variant_doc_snapshot_shard_count(variant.variant_doc_snapshot_shard_count());
 
     TabletColumn child_tcolumn;
     child_tcolumn.set_type(FieldType::OLAP_FIELD_TYPE_STRING);

be/src/vec/json/parse2column.cpp

@@ -200,8 +200,8 @@ void parse_json_to_variant(IColumn& column, const char* src, size_t length,
             }
             subcolumn->insert(std::move(values[i]), std::move(field_info));
             if (subcolumn_set.contains(paths[i].get_path())) {
-                throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "may contains duplicated entry : {}",
-                                       paths[i].get_path());
+                throw doris::Exception(ErrorCode::INVALID_ARGUMENT,
+                                       "may contains duplicated entry : {}", paths[i].get_path());
             }
             subcolumn_set.insert(paths[i].get_path());
             if (!paths[i].empty() && config.parse_to_doc_snapshot) {
@@ -215,8 +215,8 @@ void parse_json_to_variant(IColumn& column, const char* src, size_t length,
             FieldInfo field_info;
             schema_util::get_field_info(values[i], &field_info);
             if (subcolumn_set.contains(paths[i].get_path())) {
-                throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "may contains duplicated entry : {}",
-                                       paths[i].get_path());
+                throw doris::Exception(ErrorCode::INVALID_ARGUMENT,
+                                       "may contains duplicated entry : {}", paths[i].get_path());
             }
             subcolumn_set.insert(paths[i].get_path());
             if (paths[i].empty()) {

be/test/olap/rowset/segment_v2/variant_column_writer_reader_test.cpp

@@ -2881,7 +2881,6 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_doc_snapshot_mode) {
     st = it_doc_bucket->next_batch(&nrows, doc_dst);
     EXPECT_TRUE(st.ok()) << st.msg();
     EXPECT_EQ(nrows, 1000);
-    EXPECT_TRUE(assert_cast<vectorized::ColumnVariant*>(doc_dst.get())->is_doc_snapshot_mode());
 }
 
 TEST_F(VariantColumnWriterReaderTest, test_doc_snapshot_path_iterator_doc_snapshot_only) {
@@ -3007,8 +3006,8 @@ TEST_F(VariantColumnWriterReaderTest, test_doc_snapshot_path_iterator_doc_snapsh
     options.timezone = &tz;
     for (int row = 0; row < 1000; ++row) {
         std::string value;
-        assert_cast<vectorized::ColumnVariant*>(dst.get())->serialize_one_row_to_string(
-                row, &value, options);
+        assert_cast<vectorized::ColumnVariant*>(dst.get())->serialize_one_row_to_string(row, &value,
+                                                                                        options);
         EXPECT_EQ(value, "88");
     }
 }

regression-test/data/variant_p0/doc_snapshot/delete.json

@@ -0,0 +1 @@
+{"k" : 1, "__DORIS_DELETE_SIGN__": 1}