add be and fe

Author: csun5285

be/src/common/consts.h

@@ -30,7 +30,6 @@ const std::string ROW_STORE_COL = "__DORIS_ROW_STORE_COL__";
 const std::string DYNAMIC_COLUMN_NAME = "__DORIS_DYNAMIC_COL__";
 const std::string PARTIAL_UPDATE_AUTO_INC_COL = "__PARTIAL_UPDATE_AUTO_INC_COLUMN__";
 const std::string VIRTUAL_COLUMN_PREFIX = "__DORIS_VIRTUAL_COL__";
-const std::string SPARSE_COLUMN_PATH = "__DORIS_VARIANT_SPARSE__";
 
 /// The maximum precision representable by a 4-byte decimal (Decimal4Value)
 constexpr int MAX_DECIMAL32_PRECISION = 9;

be/src/olap/olap_common.h

@@ -448,6 +448,8 @@ struct OlapReaderStatistics {
     int64_t variant_subtree_leaf_iter_count = 0;
     int64_t variant_subtree_hierarchical_iter_count = 0;
     int64_t variant_subtree_sparse_iter_count = 0;
+    int64_t variant_subtree_doc_snapshot_extract_iter_count = 0;
+    int64_t variant_subtree_doc_snapshot_all_iter_count = 0;
 };
 
 using ColumnId = uint32_t;

be/src/olap/rowset/segment_creator.cpp

@@ -66,10 +66,6 @@ Status SegmentFlusher::flush_single_block(const vectorized::Block* block, int32_
         return Status::OK();
     }
     vectorized::Block flush_block(*block);
-    if (_context.write_type != DataWriteType::TYPE_COMPACTION &&
-        _context.tablet_schema->num_variant_columns() > 0) {
-        RETURN_IF_ERROR(_parse_variant_columns(flush_block));
-    }
     bool no_compression = flush_block.bytes() <= config::segment_compression_threshold_kb * 1024;
     if (config::enable_vertical_segment_writer) {
         std::unique_ptr<segment_v2::VerticalSegmentWriter> writer;
@@ -85,31 +81,6 @@ Status SegmentFlusher::flush_single_block(const vectorized::Block* block, int32_
     return Status::OK();
 }
 
-Status SegmentFlusher::_internal_parse_variant_columns(vectorized::Block& block) {
-    size_t num_rows = block.rows();
-    if (num_rows == 0) {
-        return Status::OK();
-    }
-
-    std::vector<int> variant_column_pos;
-    for (int i = 0; i < block.columns(); ++i) {
-        const auto& entry = block.get_by_position(i);
-        if (entry.type->get_primitive_type() == TYPE_VARIANT) {
-            variant_column_pos.push_back(i);
-        }
-    }
-
-    if (variant_column_pos.empty()) {
-        return Status::OK();
-    }
-
-    vectorized::ParseConfig config;
-    config.enable_flatten_nested = _context.tablet_schema->variant_flatten_nested();
-    RETURN_IF_ERROR(
-            vectorized::schema_util::parse_variant_columns(block, variant_column_pos, config));
-    return Status::OK();
-}
-
 Status SegmentFlusher::close() {
     return _seg_files.close();
 }

be/src/olap/rowset/segment_creator.h

@@ -139,11 +139,6 @@ class SegmentFlusher {
     Status create_writer(std::unique_ptr<SegmentFlusher::Writer>& writer, uint32_t segment_id);
 
 private:
-    // This method will catch exception when allocate memory failed
-    Status _parse_variant_columns(vectorized::Block& block) {
-        RETURN_IF_CATCH_EXCEPTION({ return _internal_parse_variant_columns(block); });
-    }
-    Status _internal_parse_variant_columns(vectorized::Block& block);
     Status _add_rows(std::unique_ptr<segment_v2::SegmentWriter>& segment_writer,
                      const vectorized::Block* block, size_t row_offset, size_t row_num);
     Status _add_rows(std::unique_ptr<segment_v2::VerticalSegmentWriter>& segment_writer,

be/src/olap/rowset/segment_v2/column_writer.cpp

@@ -295,6 +295,11 @@ Status ColumnWriter::create_variant_writer(const ColumnWriterOptions& opts,
                                            const TabletColumn* column, io::FileWriter* file_writer,
                                            std::unique_ptr<ColumnWriter>* writer) {
     if (column->is_extracted_column()) {
+        if (column->name().find(DOC_SNAPSHOT_COLUMN_PATH) != std::string::npos) {
+            *writer = std::make_unique<VariantCompactionDocSnapshotWriter>(
+                    opts, column, std::unique_ptr<Field>(FieldFactory::create(*column)));
+            return Status::OK();
+        }
         VLOG_DEBUG << "gen subwriter for " << column->path_info_ptr()->get_path();
         *writer = std::make_unique<VariantSubcolumnWriter>(
                 opts, column, std::unique_ptr<Field>(FieldFactory::create(*column)));

be/src/olap/rowset/segment_v2/external_col_meta_util.cpp

@@ -172,27 +172,7 @@ Status ExternalColMetaUtil::write_external_column_meta(
     // 2) write pointers via proto fields
     footer->set_col_meta_region_start(meta_region_start);
 
-    // 3) clear inline columns to enable true on-demand meta loading
-    // Note: footer->columns has already been pruned to only Top Level Columns in externalize_from_footer
-    // But for full externalization, we might want to clear them all or keep only necessary info?
-    // The original logic was footer->clear_columns().
-    // If we clear columns, the Reader needs to know how to reconstruct the schema.
-    // Currently, SegmentFooterPB.columns is used as the schema source if present.
-    // If we clear it, Reader must rely on External Meta.
-    // However, the Reader typically reads footer first. If columns is empty, it assumes V3 and reads external.
-    // So yes, we should clear it.
-    // But wait, in externalize_from_footer we carefully put Top Level columns back into footer->columns.
-    // Why? Because in previous logic, we might want to keep roots in footer?
-    // The previous logic: "replace Footer.columns with only the kept top-level columns".
-    // BUT then `write_external_column_meta` calls `footer->clear_columns()` at the end!
-    // So `footer->columns` will be empty anyway.
-    // The only reason to reconstruct `footer->columns` in `externalize_from_footer` is if `write_external_column_meta` logic depended on it.
-    // In my updated `write_external_column_meta`, I iterate over `all_metas` which is returned by `externalize_from_footer`.
-    // So I don't strictly need `footer->columns` to be correct in between.
-    // However, strictly following protocol: `externalize_from_footer` modifies footer to reflect "logical" columns (Top Level).
-    // And then `write_external_column_meta` finalizes it by clearing them and setting pointers.
-
-    footer->clear_columns();
+    // Note: footer->columns has already been pruned in externalize_from_footer
     return Status::OK();
 }
 

be/src/olap/rowset/segment_v2/segment.cpp

@@ -650,7 +650,7 @@ Status Segment::new_default_iterator(const TabletColumn& tablet_column,
 Status Segment::new_column_iterator(const TabletColumn& tablet_column,
                                     std::unique_ptr<ColumnIterator>* iter,
                                     const StorageReadOptions* opt,
-                                    const std::unordered_map<int32_t, PathToSparseColumnCacheUPtr>*
+                                    const std::unordered_map<int32_t, PathToBinaryColumnCacheUPtr>*
                                             variant_sparse_column_cache) {
     if (opt->runtime_state != nullptr) {
         _be_exec_version = opt->runtime_state->be_exec_version();
@@ -675,7 +675,7 @@ Status Segment::new_column_iterator(const TabletColumn& tablet_column,
     }
     if (reader->get_meta_type() == FieldType::OLAP_FIELD_TYPE_VARIANT) {
         // if sparse_column_cache_ptr is nullptr, means the sparse column cache is not used
-        PathToSparseColumnCache* sparse_column_cache_ptr = nullptr;
+        PathToBinaryColumnCache* sparse_column_cache_ptr = nullptr;
         if (variant_sparse_column_cache) {
             auto it = variant_sparse_column_cache->find(unique_id);
             if (it != variant_sparse_column_cache->end()) {

be/src/olap/rowset/segment_v2/segment.h

@@ -115,7 +115,7 @@ class Segment : public std::enable_shared_from_this<Segment>, public MetadataAdd
     // if variant_sparse_column_cache is nullptr, means the sparse column cache is not used
     Status new_column_iterator(const TabletColumn& tablet_column,
                                std::unique_ptr<ColumnIterator>* iter, const StorageReadOptions* opt,
-                               const std::unordered_map<int32_t, PathToSparseColumnCacheUPtr>*
+                               const std::unordered_map<int32_t, PathToBinaryColumnCacheUPtr>*
                                        variant_sparse_column_cache = nullptr);
 
     Status new_index_iterator(const TabletColumn& tablet_column, const TabletIndex* index_meta,

be/src/olap/rowset/segment_v2/segment_iterator.cpp

@@ -411,7 +411,7 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) {
             if (int32_t uid = col->get_unique_id(); !_variant_sparse_column_cache.contains(uid)) {
                 DCHECK(uid >= 0);
                 _variant_sparse_column_cache.emplace(uid,
-                                                     std::make_unique<PathToSparseColumnCache>());
+                                                     std::make_unique<PathToBinaryColumnCache>());
             }
         }
     }

be/src/olap/rowset/segment_v2/segment_iterator.h

@@ -516,7 +516,7 @@ class SegmentIterator : public RowwiseIterator {
     IndexQueryContextPtr _index_query_context;
 
     // key is column uid, value is the sparse column cache
-    std::unordered_map<int32_t, PathToSparseColumnCacheUPtr> _variant_sparse_column_cache;
+    std::unordered_map<int32_t, PathToBinaryColumnCacheUPtr> _variant_sparse_column_cache;
 
     bool _find_condition_cache = false;
     std::shared_ptr<std::vector<bool>> _condition_cache;