Merge branch 'master' into show-columns-hide-version

Author: catpineapple

.github/workflows/auto-cherry-pick.yml

@@ -31,7 +31,7 @@ permissions:
 jobs:
   auto_cherry_pick:
     runs-on: ubuntu-latest
-    if: ${{(contains(github.event.pull_request.labels.*.name, 'dev/4.0.x') || contains(github.event.pull_request.labels.*.name, 'dev/3.1.x') || github.event.label.name == 'dev/4.0.x' || github.event.label.name == 'dev/3.1.x') && github.event.pull_request.merged == true }}
+    if: ${{(contains(github.event.pull_request.labels.*.name, 'dev/4.0.x') || github.event.label.name == 'dev/4.0.x') && github.event.pull_request.merged == true }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
@@ -54,14 +54,7 @@ jobs:
             else
               echo "SHA matches: $calculated_sha"
             fi    
-      - name: Auto cherry-pick to branch-3.1
-        if: ${{ ((github.event.action == 'labeled' && github.event.label.name == 'dev/3.1.x'))|| ((github.event_name == 'pull_request_target' && github.event.action == 'closed') && contains(github.event.pull_request.labels.*.name, 'dev/3.1.x')) }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          REPO_NAME: ${{ github.repository }}
-          CONFLICT_LABEL: dev/3.1.x-conflict
-        run: |
-            python tools/auto-pick-script.py ${{ github.event.pull_request.number }} branch-3.1    
+
       - name: Auto cherry-pick to branch-4.0
         if: ${{ ((github.event.action == 'labeled' && github.event.label.name == 'dev/4.0.x'))|| ((github.event_name == 'pull_request_target' && github.event.action == 'closed') && contains(github.event.pull_request.labels.*.name, 'dev/4.0.x')) }}
         env:

NOTICE.txt

@@ -1,5 +1,5 @@
 Apache Doris
-Copyright 2018-2025 The Apache Software Foundation
+Copyright 2018-2026 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

be/src/agent/cgroup_cpu_ctl.cpp

@@ -234,10 +234,11 @@ Status CgroupCpuCtl::write_cg_sys_file(std::string file_path, std::string value,
     auto str = fmt::format("{}\n", value);
     ssize_t ret = write(fd, str.c_str(), str.size());
     if (ret == -1) {
-        LOG(ERROR) << msg << " write sys file failed";
-        return Status::InternalError<false>("{} write sys file failed", msg);
+        LOG(ERROR) << msg << " write sys file failed, file_path=" << file_path;
+        return Status::InternalError<false>("{} write sys file failed, file_path={}", msg,
+                                            file_path);
     }
-    LOG(INFO) << msg << " success";
+    LOG(INFO) << msg << " success, file path: " << file_path;
     return Status::OK();
 }
 

be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp

@@ -224,6 +224,12 @@ Status CloudTabletCalcDeleteBitmapTask::handle() const {
     });
     Status status;
     if (_sub_txn_ids.empty()) {
+        // Check empty rowset for non-sub_txn case
+        if (_engine.txn_delete_bitmap_cache().is_empty_rowset(_transaction_id, _tablet_id)) {
+            LOG(INFO) << "tablet=" << _tablet_id << ", txn=" << _transaction_id
+                      << " is empty rowset, skip delete bitmap calculation";
+            return Status::OK();
+        }
         status = _handle_rowset(tablet, _version);
     } else {
         std::stringstream ss;
@@ -237,9 +243,18 @@ Status CloudTabletCalcDeleteBitmapTask::handle() const {
         std::vector<RowsetSharedPtr> invisible_rowsets;
         DeleteBitmapPtr tablet_delete_bitmap =
                 std::make_shared<DeleteBitmap>(tablet->tablet_meta()->delete_bitmap());
-        for (int i = 0; i < _sub_txn_ids.size(); ++i) {
+        size_t empty_rowset_count = 0;
+        for (size_t i = 0; i < _sub_txn_ids.size(); ++i) {
             int64_t sub_txn_id = _sub_txn_ids[i];
             int64_t version = _version + i;
+            // Check empty rowset for each sub_txn using sub_txn_id
+            if (_engine.txn_delete_bitmap_cache().is_empty_rowset(sub_txn_id, _tablet_id)) {
+                LOG(INFO) << "tablet=" << _tablet_id << ", sub_txn=" << sub_txn_id
+                          << ", version=" << version
+                          << " is empty rowset, skip delete bitmap calculation";
+                empty_rowset_count++;
+                continue;
+            }
             LOG(INFO) << "start calc delete bitmap for txn_id=" << _transaction_id
                       << ", sub_txn_id=" << sub_txn_id << ", table_id=" << tablet->table_id()
                       << ", partition_id=" << tablet->partition_id() << ", tablet_id=" << _tablet_id
@@ -254,7 +269,7 @@ Status CloudTabletCalcDeleteBitmapTask::handle() const {
                           << ", cur_version=" << version << ", status=" << status;
                 return status;
             }
-            DCHECK(invisible_rowsets.size() == i + 1);
+            DCHECK(invisible_rowsets.size() == i + 1 - empty_rowset_count);
         }
     }
     DBUG_EXECUTE_IF("CloudCalcDbmTask.handle.return.block",

be/src/cloud/cloud_rowset_builder.cpp

@@ -127,6 +127,15 @@ const RowsetMetaSharedPtr& CloudRowsetBuilder::rowset_meta() {
 
 Status CloudRowsetBuilder::set_txn_related_delete_bitmap() {
     if (_tablet->enable_unique_key_merge_on_write()) {
+        // For empty rowsets when skip_writing_empty_rowset_metadata=true,
+        // store only a lightweight marker instead of full rowset info.
+        // This allows CalcDeleteBitmapTask to detect and skip gracefully,
+        // while using minimal memory (~16 bytes per entry).
+        if (_skip_writing_rowset_metadata) {
+            _engine.txn_delete_bitmap_cache().mark_empty_rowset(_req.txn_id, _tablet->tablet_id(),
+                                                                _req.txn_expiration);
+            return Status::OK();
+        }
         if (config::enable_merge_on_write_correctness_check && _rowset->num_rows() != 0) {
             auto st = _tablet->check_delete_bitmap_correctness(
                     _delete_bitmap, _rowset->end_version() - 1, _req.txn_id, *_rowset_ids);

be/src/cloud/cloud_txn_delete_bitmap_cache.cpp

@@ -231,7 +231,9 @@ void CloudTxnDeleteBitmapCache::remove_expired_tablet_txn_info() {
     std::unique_lock<std::shared_mutex> wlock(_rwlock);
     while (!_expiration_txn.empty()) {
         auto iter = _expiration_txn.begin();
-        if (_txn_map.find(iter->second) == _txn_map.end()) {
+        bool in_txn_map = _txn_map.find(iter->second) != _txn_map.end();
+        bool in_markers = _empty_rowset_markers.find(iter->second) != _empty_rowset_markers.end();
+        if (!in_txn_map && !in_markers) {
             _expiration_txn.erase(iter);
             continue;
         }
@@ -241,6 +243,7 @@ void CloudTxnDeleteBitmapCache::remove_expired_tablet_txn_info() {
         if (iter->first > current_time) {
             break;
         }
+        // Clean from _txn_map if exists
         auto txn_iter = _txn_map.find(iter->second);
         if ((txn_iter != _txn_map.end()) && (iter->first == txn_iter->second.txn_expiration)) {
             LOG_INFO("clean expired delete bitmap")
@@ -253,6 +256,14 @@ void CloudTxnDeleteBitmapCache::remove_expired_tablet_txn_info() {
             erase(cache_key);
             _txn_map.erase(iter->second);
         }
+        // Clean from _empty_rowset_markers if exists
+        auto marker_iter = _empty_rowset_markers.find(iter->second);
+        if (marker_iter != _empty_rowset_markers.end()) {
+            LOG_INFO("clean expired empty rowset marker")
+                    .tag("txn_id", iter->second.txn_id)
+                    .tag("tablet_id", iter->second.tablet_id);
+            _empty_rowset_markers.erase(marker_iter);
+        }
         _expiration_txn.erase(iter);
     }
 }
@@ -274,6 +285,32 @@ void CloudTxnDeleteBitmapCache::remove_unused_tablet_txn_info(TTransactionId tra
     }
 }
 
+void CloudTxnDeleteBitmapCache::mark_empty_rowset(TTransactionId txn_id, int64_t tablet_id,
+                                                  int64_t txn_expiration) {
+    int64_t txn_expiration_min =
+            duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch())
+                    .count() +
+            config::tablet_txn_info_min_expired_seconds;
+    txn_expiration = std::max(txn_expiration_min, txn_expiration);
+
+    if (config::enable_mow_verbose_log) {
+        LOG_INFO("mark empty rowset")
+                .tag("txn_id", txn_id)
+                .tag("tablet_id", tablet_id)
+                .tag("expiration", txn_expiration);
+    }
+    std::unique_lock<std::shared_mutex> wlock(_rwlock);
+    TxnKey txn_key(txn_id, tablet_id);
+    _empty_rowset_markers.emplace(txn_key);
+    _expiration_txn.emplace(txn_expiration, txn_key);
+}
+
+bool CloudTxnDeleteBitmapCache::is_empty_rowset(TTransactionId txn_id, int64_t tablet_id) {
+    std::shared_lock<std::shared_mutex> rlock(_rwlock);
+    TxnKey txn_key(txn_id, tablet_id);
+    return _empty_rowset_markers.contains(txn_key);
+}
+
 void CloudTxnDeleteBitmapCache::_clean_thread_callback() {
     do {
         remove_expired_tablet_txn_info();

be/src/cloud/cloud_txn_delete_bitmap_cache.h

@@ -59,6 +59,16 @@ class CloudTxnDeleteBitmapCache : public LRUCachePolicy {
 
     void remove_unused_tablet_txn_info(TTransactionId transaction_id, int64_t tablet_id);
 
+    // Mark a rowset as empty/skipped (lightweight marker, no rowset stored)
+    // Used for empty rowsets when skip_writing_empty_rowset_metadata is enabled
+    void mark_empty_rowset(TTransactionId txn_id, int64_t tablet_id, int64_t txn_expiration);
+
+    // Check if this is a known empty/skipped rowset
+    // Returns true if was marked as empty rowset
+    // Note: Does not remove the marker, as CalcDeleteBitmapTask may retry.
+    // Cleanup is handled by expiration-based removal in remove_expired_tablet_txn_info()
+    bool is_empty_rowset(TTransactionId txn_id, int64_t tablet_id);
+
     // !!!ATTENTION!!!: the delete bitmap stored in CloudTxnDeleteBitmapCache contains sentinel marks,
     // and the version in BitmapKey is DeleteBitmap::TEMP_VERSION_COMMON.
     // when using delete bitmap from this cache, the caller should manually remove these marks if don't need it
@@ -107,6 +117,9 @@ class CloudTxnDeleteBitmapCache : public LRUCachePolicy {
 
     std::map<TxnKey, TxnVal> _txn_map;
     std::multimap<int64_t, TxnKey> _expiration_txn;
+    // Lightweight markers for empty/skipped rowsets (only stores TxnKey, ~16 bytes per entry)
+    // Used to track empty rowsets that were not committed to meta-service
+    std::set<TxnKey> _empty_rowset_markers;
     std::shared_mutex _rwlock;
     std::shared_ptr<Thread> _clean_thread;
     CountDownLatch _stop_latch;

be/src/cloud/config.cpp

@@ -135,7 +135,7 @@ DEFINE_mInt64(warm_up_rowset_sync_wait_max_timeout_ms, "120000");
 DEFINE_mBool(enable_warmup_immediately_on_new_rowset, "false");
 
 // Packed file manager config
-DEFINE_mBool(enable_packed_file, "true");
+DEFINE_mBool(enable_packed_file, "false");
 DEFINE_mInt64(packed_file_size_threshold_bytes, "5242880"); // 5MB
 DEFINE_mInt64(packed_file_time_threshold_ms, "100");        // 100ms
 DEFINE_mInt64(packed_file_try_lock_timeout_ms, "5");        // 5ms

be/src/cloud/pb_convert.cpp

@@ -80,6 +80,7 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, const RowsetMetaPB& in)
     out->set_txn_expiration(in.txn_expiration());
     out->set_segments_overlap_pb(in.segments_overlap_pb());
     out->set_segments_key_bounds_truncated(in.segments_key_bounds_truncated());
+    out->mutable_num_segment_rows()->CopyFrom(in.num_segment_rows());
     out->mutable_segments_file_size()->CopyFrom(in.segments_file_size());
     out->set_index_id(in.index_id());
     if (in.has_schema_version()) {
@@ -157,6 +158,7 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, RowsetMetaPB&& in) {
     out->set_txn_expiration(in.txn_expiration());
     out->set_segments_overlap_pb(in.segments_overlap_pb());
     out->set_segments_key_bounds_truncated(in.segments_key_bounds_truncated());
+    out->mutable_num_segment_rows()->Swap(in.mutable_num_segment_rows());
     out->mutable_segments_file_size()->Swap(in.mutable_segments_file_size());
     out->set_index_id(in.index_id());
     if (in.has_schema_version()) {
@@ -246,6 +248,7 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, const RowsetMetaCloudPB& in)
     out->set_txn_expiration(in.txn_expiration());
     out->set_segments_overlap_pb(in.segments_overlap_pb());
     out->set_segments_key_bounds_truncated(in.segments_key_bounds_truncated());
+    out->mutable_num_segment_rows()->CopyFrom(in.num_segment_rows());
     out->mutable_segments_file_size()->CopyFrom(in.segments_file_size());
     out->set_index_id(in.index_id());
     if (in.has_schema_version()) {
@@ -323,6 +326,7 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, RowsetMetaCloudPB&& in) {
     out->set_txn_expiration(in.txn_expiration());
     out->set_segments_overlap_pb(in.segments_overlap_pb());
     out->set_segments_key_bounds_truncated(in.segments_key_bounds_truncated());
+    out->mutable_num_segment_rows()->Swap(in.mutable_num_segment_rows());
     out->mutable_segments_file_size()->Swap(in.mutable_segments_file_size());
     out->set_index_id(in.index_id());
     if (in.has_schema_version()) {

be/src/common/config.cpp

@@ -384,6 +384,12 @@ DEFINE_mInt32(trash_file_expire_time_sec, "0");
 // modify them upon necessity
 DEFINE_Int32(min_file_descriptor_number, "60000");
 DEFINE_mBool(disable_segment_cache, "false");
+// Enable checking segment rows consistency between rowset meta and segment footer
+DEFINE_mBool(enable_segment_rows_consistency_check, "false");
+DEFINE_mBool(enable_segment_rows_check_core, "false");
+// ATTENTION: For test only. In test environment, there are no historical data,
+// so all rowset meta should have segment rows info.
+DEFINE_mBool(fail_when_segment_rows_not_in_rowset_meta, "false");
 DEFINE_String(row_cache_mem_limit, "20%");
 
 // Cache for storage page size
@@ -567,6 +573,14 @@ DEFINE_Bool(enable_all_http_auth, "false");
 // Number of webserver workers
 DEFINE_Int32(webserver_num_workers, "128");
 
+// Async replies: stream load only now
+// reply wait timeout only happens if:
+// 1. Stream load fragment execution times out
+//    HTTP request freed → stream load canceled
+// 2. Client disconnects
+DEFINE_mInt32(async_reply_timeout_s, "60");
+DEFINE_Validator(async_reply_timeout_s, [](const int config) -> bool { return config >= 3; });
+
 DEFINE_Bool(enable_single_replica_load, "true");
 // Number of download workers for single replica load
 DEFINE_Int32(single_replica_load_download_num_workers, "64");
@@ -1010,7 +1024,7 @@ DEFINE_mInt64(big_column_size_buffer, "65535");
 DEFINE_mInt64(small_column_size_buffer, "100");
 
 // Perform the always_true check at intervals determined by runtime_filter_sampling_frequency
-DEFINE_mInt32(runtime_filter_sampling_frequency, "64");
+DEFINE_mInt32(runtime_filter_sampling_frequency, "32");
 DEFINE_mInt32(execution_max_rpc_timeout_sec, "3600");
 DEFINE_mBool(execution_ignore_eovercrowded, "true");
 // cooldown task configs
@@ -1234,7 +1248,7 @@ DEFINE_Int32(segment_cache_capacity, "-1");
 DEFINE_Int32(segment_cache_fd_percentage, "20");
 DEFINE_mInt32(estimated_mem_per_column_reader, "512");
 DEFINE_Int32(segment_cache_memory_percentage, "5");
-DEFINE_Bool(enable_segment_cache_prune, "true");
+DEFINE_Bool(enable_segment_cache_prune, "false");
 
 // enable feature binlog, default false
 DEFINE_Bool(enable_feature_binlog, "false");
@@ -1599,18 +1613,7 @@ DEFINE_mInt64(max_csv_line_reader_output_buffer_size, "4294967296");
 // Maximum number of OpenMP threads allowed for concurrent vector index builds.
 // -1 means auto: use 80% of the available CPU cores.
 DEFINE_Int32(omp_threads_limit, "-1");
-DEFINE_Validator(omp_threads_limit, [](const int config) -> bool {
-    if (config > 0) {
-        omp_threads_limit = config;
-        return true;
-    }
-    CpuInfo::init();
-    int core_cap = config::num_cores > 0 ? config::num_cores : CpuInfo::num_cores();
-    core_cap = std::max(1, core_cap);
-    // Use at most 80% of the available CPU cores.
-    omp_threads_limit = std::max(1, core_cap * 4 / 5);
-    return true;
-});
+
 // The capacity of segment partial column cache, used to cache column readers for each segment.
 DEFINE_mInt32(max_segment_partial_column_cache_size, "100");
 
@@ -2085,6 +2088,8 @@ Status set_fuzzy_configs() {
             ((distribution(*generator) % 2) == 0) ? "10" : "4294967295";
     fuzzy_field_and_value["skip_writing_empty_rowset_metadata"] =
             ((distribution(*generator) % 2) == 0) ? "true" : "false";
+    fuzzy_field_and_value["enable_packed_file"] =
+            ((distribution(*generator) % 2) == 0) ? "true" : "false";
     fuzzy_field_and_value["max_segment_partial_column_cache_size"] =
             ((distribution(*generator) % 2) == 0) ? "5" : "10";