Merge branch 'apache:master' into sss

Author: Tech-Circle-48

.asf.yaml

@@ -63,6 +63,7 @@ github:
           - COMPILE (DORIS_COMPILE)
           - Need_2_Approval
           - Cloud UT (Doris Cloud UT)
+          - performance (Doris Performance)
 
       required_pull_request_reviews:
         dismiss_stale_reviews: true

README.md

@@ -59,12 +59,9 @@ Apache Doris is an easy-to-use, high-performance and real-time analytical databa
 
 All this makes Apache Doris an ideal tool for scenarios including report analysis, ad-hoc query, unified data warehouse, and data lake query acceleration. On Apache Doris, users can build various applications, such as user behavior analysis, AB test platform, log retrieval analysis, user portrait analysis, and order analysis.
 
-🎉 Version 2.1.4 released now. Check out the 🔗[Release Notes](https://doris.apache.org/docs/releasenotes/release-2.1.4) here. The 2.1 verison delivers exceptional performance with 100% higher out-of-the-box queries proven by TPC-DS 1TB tests, enhanced data lake analytics that are 4-6 times speedier than Trino and Spark, solid support for semi-structured data analysis with new Variant types and suite of analytical functions, asynchronous materialized views for query acceleration, optimized real-time writing at scale, and better workload management with stability and runtime SQL resource tracking.
+🎉 Check out the 🔗[All releases](https://doris.apache.org/docs/releasenotes/all-release), where you'll find a chronological summary of Apache Doris versions released over the past year.
 
-
-🎉 Version 2.0.12 is now released ! This fully evolved and stable release is ready for all users to upgrade. Check out the 🔗[Release Notes](https://doris.apache.org/docs/2.0/releasenotes/release-2.0.12) here. 
-
-👀 Have a look at the 🔗[Official Website](https://doris.apache.org/) for a comprehensive list of Apache Doris's core features, blogs and user cases.
+👀 Explore the 🔗[Official Website](https://doris.apache.org/) to discover Apache Doris's core features, blogs, and user cases in detail.
 
 ## 📈 Usage Scenarios
 

be/src/olap/base_tablet.cpp

@@ -28,6 +28,7 @@
 #include "common/status.h"
 #include "olap/calc_delete_bitmap_executor.h"
 #include "olap/delete_bitmap_calculator.h"
+#include "olap/iterators.h"
 #include "olap/memtable.h"
 #include "olap/partial_update_info.h"
 #include "olap/primary_key_index.h"
@@ -81,7 +82,9 @@ Status _get_segment_column_iterator(const BetaRowsetSharedPtr& rowset, uint32_t
                                             rowset->rowset_id().to_string(), segid));
     }
     segment_v2::SegmentSharedPtr segment = *it;
-    RETURN_IF_ERROR(segment->new_column_iterator(target_column, column_iterator, nullptr));
+    StorageReadOptions opts;
+    opts.stats = stats;
+    RETURN_IF_ERROR(segment->new_column_iterator(target_column, column_iterator, &opts));
     segment_v2::ColumnIteratorOptions opt {
             .use_page_cache = !config::disable_storage_page_cache,
             .file_reader = segment->file_reader().get(),
@@ -496,7 +499,7 @@ Status BaseTablet::lookup_row_key(const Slice& encoded_key, TabletSchema* latest
 
         for (auto id : picked_segments) {
             Status s = segments[id]->lookup_row_key(encoded_key, schema, with_seq_col, with_rowid,
-                                                    &loc, encoded_seq_value, stats);
+                                                    &loc, stats, encoded_seq_value);
             if (s.is<KEY_NOT_FOUND>()) {
                 continue;
             }
@@ -612,7 +615,7 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
     vectorized::Block ordered_block = block.clone_empty();
     uint32_t pos = 0;
 
-    RETURN_IF_ERROR(seg->load_pk_index_and_bf()); // We need index blocks to iterate
+    RETURN_IF_ERROR(seg->load_pk_index_and_bf(nullptr)); // We need index blocks to iterate
     const auto* pk_idx = seg->get_primary_key_index();
     int total = pk_idx->num_rows();
     uint32_t row_id = 0;
@@ -626,7 +629,7 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
     std::vector<std::unique_ptr<SegmentCacheHandle>> segment_caches(specified_rowsets.size());
     while (remaining > 0) {
         std::unique_ptr<segment_v2::IndexedColumnIterator> iter;
-        RETURN_IF_ERROR(pk_idx->new_iterator(&iter));
+        RETURN_IF_ERROR(pk_idx->new_iterator(&iter, nullptr));
 
         size_t num_to_read = std::min(batch_size, remaining);
         auto index_type = vectorized::DataTypeFactory::instance().create_data_type(

be/src/olap/delete_bitmap_calculator.cpp

@@ -145,12 +145,11 @@ Status MergeIndexDeleteBitmapCalculator::init(RowsetId rowset_id,
                 MergeIndexDeleteBitmapCalculatorContext::Comparator(seq_col_length, _rowid_length);
         _contexts.reserve(segments.size());
         _heap = std::make_unique<Heap>(_comparator);
-
         for (auto& segment : segments) {
-            RETURN_IF_ERROR(segment->load_index());
+            RETURN_IF_ERROR(segment->load_index(nullptr));
             auto pk_idx = segment->get_primary_key_index();
             std::unique_ptr<segment_v2::IndexedColumnIterator> index;
-            RETURN_IF_ERROR(pk_idx->new_iterator(&index));
+            RETURN_IF_ERROR(pk_idx->new_iterator(&index, nullptr));
             auto index_type = vectorized::DataTypeFactory::instance().create_data_type(
                     pk_idx->type_info()->type(), 1, 0);
             _contexts.emplace_back(std::move(index), index_type, segment->id(), pk_idx->num_rows());

be/src/olap/olap_common.h

@@ -389,6 +389,30 @@ struct OlapReaderStatistics {
     int64_t collect_iterator_merge_next_timer = 0;
     int64_t collect_iterator_normal_next_timer = 0;
     int64_t delete_bitmap_get_agg_ns = 0;
+
+    int64_t tablet_reader_init_timer_ns = 0;
+    int64_t tablet_reader_capture_rs_readers_timer_ns = 0;
+    int64_t tablet_reader_init_return_columns_timer_ns = 0;
+    int64_t tablet_reader_init_keys_param_timer_ns = 0;
+    int64_t tablet_reader_init_orderby_keys_param_timer_ns = 0;
+    int64_t tablet_reader_init_conditions_param_timer_ns = 0;
+    int64_t tablet_reader_init_delete_condition_param_timer_ns = 0;
+    int64_t block_reader_vcollect_iter_init_timer_ns = 0;
+    int64_t block_reader_rs_readers_init_timer_ns = 0;
+    int64_t block_reader_build_heap_init_timer_ns = 0;
+
+    int64_t rowset_reader_get_segment_iterators_timer_ns = 0;
+    int64_t rowset_reader_create_iterators_timer_ns = 0;
+    int64_t rowset_reader_init_iterators_timer_ns = 0;
+    int64_t rowset_reader_load_segments_timer_ns = 0;
+
+    int64_t segment_iterator_init_timer_ns = 0;
+    int64_t segment_iterator_init_return_column_iterators_timer_ns = 0;
+    int64_t segment_iterator_init_bitmap_index_iterators_timer_ns = 0;
+    int64_t segment_iterator_init_inverted_index_iterators_timer_ns = 0;
+
+    int64_t segment_create_column_readers_timer_ns = 0;
+    int64_t segment_load_index_timer_ns = 0;
 };
 
 using ColumnId = uint32_t;

be/src/olap/primary_key_index.cpp

@@ -95,27 +95,29 @@ Status PrimaryKeyIndexBuilder::finalize(segment_v2::PrimaryKeyIndexMetaPB* meta)
 }
 
 Status PrimaryKeyIndexReader::parse_index(io::FileReaderSPtr file_reader,
-                                          const segment_v2::PrimaryKeyIndexMetaPB& meta) {
+                                          const segment_v2::PrimaryKeyIndexMetaPB& meta,
+                                          OlapReaderStatistics* pk_index_load_stats) {
     // parse primary key index
     _index_reader.reset(new segment_v2::IndexedColumnReader(file_reader, meta.primary_key_index()));
     _index_reader->set_is_pk_index(true);
     RETURN_IF_ERROR(_index_reader->load(!config::disable_pk_storage_page_cache, false,
-                                        _pk_index_load_stats));
+                                        pk_index_load_stats));
 
     _index_parsed = true;
     return Status::OK();
 }
 
 Status PrimaryKeyIndexReader::parse_bf(io::FileReaderSPtr file_reader,
-                                       const segment_v2::PrimaryKeyIndexMetaPB& meta) {
+                                       const segment_v2::PrimaryKeyIndexMetaPB& meta,
+                                       OlapReaderStatistics* pk_index_load_stats) {
     // parse bloom filter
     segment_v2::ColumnIndexMetaPB column_index_meta = meta.bloom_filter_index();
     segment_v2::BloomFilterIndexReader bf_index_reader(std::move(file_reader),
                                                        column_index_meta.bloom_filter_index());
     RETURN_IF_ERROR(bf_index_reader.load(!config::disable_pk_storage_page_cache, false,
-                                         _pk_index_load_stats));
+                                         pk_index_load_stats));
     std::unique_ptr<segment_v2::BloomFilterIndexIterator> bf_iter;
-    RETURN_IF_ERROR(bf_index_reader.new_iterator(&bf_iter));
+    RETURN_IF_ERROR(bf_index_reader.new_iterator(&bf_iter, pk_index_load_stats));
     RETURN_IF_ERROR(bf_iter->read_bloom_filter(0, &_bf));
     segment_v2::g_pk_total_bloom_filter_num << 1;
     segment_v2::g_pk_total_bloom_filter_total_bytes << _bf->size();

be/src/olap/primary_key_index.h

@@ -98,8 +98,7 @@ class PrimaryKeyIndexBuilder {
 
 class PrimaryKeyIndexReader {
 public:
-    PrimaryKeyIndexReader(OlapReaderStatistics* pk_index_load_stats = nullptr)
-            : _index_parsed(false), _bf_parsed(false), _pk_index_load_stats(pk_index_load_stats) {}
+    PrimaryKeyIndexReader() : _index_parsed(false), _bf_parsed(false) {}
 
     ~PrimaryKeyIndexReader() {
         segment_v2::g_pk_total_bloom_filter_num << -static_cast<int64_t>(_bf_num);
@@ -109,12 +108,14 @@ class PrimaryKeyIndexReader {
     }
 
     Status parse_index(io::FileReaderSPtr file_reader,
-                       const segment_v2::PrimaryKeyIndexMetaPB& meta);
+                       const segment_v2::PrimaryKeyIndexMetaPB& meta,
+                       OlapReaderStatistics* pk_index_load_stats);
 
-    Status parse_bf(io::FileReaderSPtr file_reader, const segment_v2::PrimaryKeyIndexMetaPB& meta);
+    Status parse_bf(io::FileReaderSPtr file_reader, const segment_v2::PrimaryKeyIndexMetaPB& meta,
+                    OlapReaderStatistics* pk_index_load_stats);
 
     Status new_iterator(std::unique_ptr<segment_v2::IndexedColumnIterator>* index_iterator,
-                        OlapReaderStatistics* stats = nullptr) const {
+                        OlapReaderStatistics* stats) const {
         DCHECK(_index_parsed);
         index_iterator->reset(new segment_v2::IndexedColumnIterator(_index_reader.get(), stats));
         return Status::OK();
@@ -155,7 +156,6 @@ class PrimaryKeyIndexReader {
     std::unique_ptr<segment_v2::BloomFilter> _bf;
     size_t _bf_num = 0;
     uint64 _bf_bytes = 0;
-    OlapReaderStatistics* _pk_index_load_stats = nullptr;
 };
 
 } // namespace doris

be/src/olap/rowset/beta_rowset_reader.cpp

@@ -78,7 +78,6 @@ bool BetaRowsetReader::update_profile(RuntimeProfile* profile) {
 Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context,
                                                std::vector<RowwiseIteratorUPtr>* out_iters,
                                                bool use_cache) {
-    RETURN_IF_ERROR(_rowset->load());
     _read_context = read_context;
     // The segment iterator is created with its own statistics,
     // and the member variable '_stats'  is initialized by '_stats(&owned_stats)'.
@@ -92,6 +91,9 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
     if (_read_context->stats != nullptr) {
         _stats = _read_context->stats;
     }
+    SCOPED_RAW_TIMER(&_stats->rowset_reader_get_segment_iterators_timer_ns);
+
+    RETURN_IF_ERROR(_rowset->load());
 
     // convert RowsetReaderContext to StorageReadOptions
     _read_options.block_row_max = read_context->batch_size;
@@ -225,9 +227,12 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
     bool should_use_cache = use_cache || (_read_context->reader_type == ReaderType::READER_QUERY &&
                                           enable_segment_cache);
     SegmentCacheHandle segment_cache_handle;
-    RETURN_IF_ERROR(SegmentLoader::instance()->load_segments(_rowset, &segment_cache_handle,
-                                                             should_use_cache,
-                                                             /*need_load_pk_index_and_bf*/ false));
+    {
+        SCOPED_RAW_TIMER(&_stats->rowset_reader_load_segments_timer_ns);
+        RETURN_IF_ERROR(SegmentLoader::instance()->load_segments(
+                _rowset, &segment_cache_handle, should_use_cache,
+                /*need_load_pk_index_and_bf*/ false));
+    }
 
     // create iterator for each segment
     auto& segments = segment_cache_handle.get_segments();
@@ -253,6 +258,7 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
     const bool use_lazy_init_iterators =
             !is_merge_iterator && _read_context->reader_type == ReaderType::READER_QUERY;
     for (int i = seg_start; i < seg_end; i++) {
+        SCOPED_RAW_TIMER(&_stats->rowset_reader_create_iterators_timer_ns);
         auto& seg_ptr = segments[i];
         std::unique_ptr<RowwiseIterator> iter;
 
@@ -317,6 +323,8 @@ Status BetaRowsetReader::_init_iterator() {
     std::vector<RowwiseIteratorUPtr> iterators;
     RETURN_IF_ERROR(get_segment_iterators(_read_context, &iterators));
 
+    SCOPED_RAW_TIMER(&_stats->rowset_reader_init_iterators_timer_ns);
+
     if (_read_context->merged_rows == nullptr) {
         _read_context->merged_rows = &_merged_rows;
     }
@@ -352,8 +360,8 @@ Status BetaRowsetReader::_init_iterator() {
 }
 
 Status BetaRowsetReader::next_block(vectorized::Block* block) {
-    SCOPED_RAW_TIMER(&_stats->block_fetch_ns);
     RETURN_IF_ERROR(_init_iterator_once());
+    SCOPED_RAW_TIMER(&_stats->block_fetch_ns);
     if (_empty) {
         return Status::Error<END_OF_FILE>("BetaRowsetReader is empty");
     }
@@ -381,9 +389,8 @@ Status BetaRowsetReader::next_block(vectorized::Block* block) {
 }
 
 Status BetaRowsetReader::next_block_view(vectorized::BlockView* block_view) {
-    SCOPED_RAW_TIMER(&_stats->block_fetch_ns);
     RETURN_IF_ERROR(_init_iterator_once());
-
+    SCOPED_RAW_TIMER(&_stats->block_fetch_ns);
     RuntimeState* runtime_state = nullptr;
     if (_read_context != nullptr) {
         runtime_state = _read_context->runtime_state;

be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp

@@ -34,9 +34,8 @@ namespace segment_v2 {
 Status BloomFilterIndexReader::load(bool use_page_cache, bool kept_in_memory,
                                     OlapReaderStatistics* index_load_stats) {
     // TODO yyq: implement a new once flag to avoid status construct.
-    _index_load_stats = index_load_stats;
-    return _load_once.call([this, use_page_cache, kept_in_memory] {
-        return _load(use_page_cache, kept_in_memory);
+    return _load_once.call([this, use_page_cache, kept_in_memory, index_load_stats] {
+        return _load(use_page_cache, kept_in_memory, index_load_stats);
     });
 }
 
@@ -45,20 +44,22 @@ int64_t BloomFilterIndexReader::get_metadata_size() const {
            (_bloom_filter_index_meta ? _bloom_filter_index_meta->ByteSizeLong() : 0);
 }
 
-Status BloomFilterIndexReader::_load(bool use_page_cache, bool kept_in_memory) {
+Status BloomFilterIndexReader::_load(bool use_page_cache, bool kept_in_memory,
+                                     OlapReaderStatistics* index_load_stats) {
     const IndexedColumnMetaPB& bf_index_meta = _bloom_filter_index_meta->bloom_filter();
 
     _bloom_filter_reader.reset(new IndexedColumnReader(_file_reader, bf_index_meta));
-    RETURN_IF_ERROR(_bloom_filter_reader->load(use_page_cache, kept_in_memory, _index_load_stats));
+    RETURN_IF_ERROR(_bloom_filter_reader->load(use_page_cache, kept_in_memory, index_load_stats));
     update_metadata_size();
     return Status::OK();
 }
 
-Status BloomFilterIndexReader::new_iterator(std::unique_ptr<BloomFilterIndexIterator>* iterator) {
+Status BloomFilterIndexReader::new_iterator(std::unique_ptr<BloomFilterIndexIterator>* iterator,
+                                            OlapReaderStatistics* index_load_stats) {
     DBUG_EXECUTE_IF("BloomFilterIndexReader::new_iterator.fail", {
         return Status::InternalError("new_iterator for bloom filter index failed");
     });
-    iterator->reset(new BloomFilterIndexIterator(this));
+    iterator->reset(new BloomFilterIndexIterator(this, index_load_stats));
     return Status::OK();
 }
 

be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h

@@ -48,17 +48,18 @@ class BloomFilterIndexReader : public MetadataAdder<BloomFilterIndexReader> {
     }
 
     Status load(bool use_page_cache, bool kept_in_memory,
-                OlapReaderStatistics* _bf_index_load_stats = nullptr);
+                OlapReaderStatistics* bf_index_load_stats);
 
     BloomFilterAlgorithmPB algorithm() { return _bloom_filter_index_meta->algorithm(); }
 
     // create a new column iterator.
-    Status new_iterator(std::unique_ptr<BloomFilterIndexIterator>* iterator);
+    Status new_iterator(std::unique_ptr<BloomFilterIndexIterator>* iterator,
+                        OlapReaderStatistics* index_load_stats);
 
     const TypeInfo* type_info() const { return _type_info; }
 
 private:
-    Status _load(bool use_page_cache, bool kept_in_memory);
+    Status _load(bool use_page_cache, bool kept_in_memory, OlapReaderStatistics* index_load_stats);
 
     int64_t get_metadata_size() const override;
 
@@ -70,13 +71,12 @@ class BloomFilterIndexReader : public MetadataAdder<BloomFilterIndexReader> {
     const TypeInfo* _type_info = nullptr;
     std::unique_ptr<BloomFilterIndexPB> _bloom_filter_index_meta = nullptr;
     std::unique_ptr<IndexedColumnReader> _bloom_filter_reader;
-    OlapReaderStatistics* _index_load_stats = nullptr;
 };
 
 class BloomFilterIndexIterator {
 public:
-    explicit BloomFilterIndexIterator(BloomFilterIndexReader* reader)
-            : _reader(reader), _bloom_filter_iter(reader->_bloom_filter_reader.get()) {}
+    explicit BloomFilterIndexIterator(BloomFilterIndexReader* reader, OlapReaderStatistics* stats)
+            : _reader(reader), _bloom_filter_iter(reader->_bloom_filter_reader.get(), stats) {}
 
     // Read bloom filter at the given ordinal into `bf`.
     Status read_bloom_filter(rowid_t ordinal, std::unique_ptr<BloomFilter>* bf);