Merge branch 'master' into file_cache_admission_control

Author: xuchenhao

.github/workflows/auto-cherry-pick.yml

@@ -31,7 +31,7 @@ permissions:
 jobs:
   auto_cherry_pick:
     runs-on: ubuntu-latest
-    if: ${{(contains(github.event.pull_request.labels.*.name, 'dev/4.0.x') || contains(github.event.pull_request.labels.*.name, 'dev/3.1.x') || github.event.label.name == 'dev/4.0.x' || github.event.label.name == 'dev/3.1.x') && github.event.pull_request.merged == true }}
+    if: ${{(contains(github.event.pull_request.labels.*.name, 'dev/4.0.x') || github.event.label.name == 'dev/4.0.x') && github.event.pull_request.merged == true }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
@@ -54,14 +54,7 @@ jobs:
             else
               echo "SHA matches: $calculated_sha"
             fi    
-      - name: Auto cherry-pick to branch-3.1
-        if: ${{ ((github.event.action == 'labeled' && github.event.label.name == 'dev/3.1.x'))|| ((github.event_name == 'pull_request_target' && github.event.action == 'closed') && contains(github.event.pull_request.labels.*.name, 'dev/3.1.x')) }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          REPO_NAME: ${{ github.repository }}
-          CONFLICT_LABEL: dev/3.1.x-conflict
-        run: |
-            python tools/auto-pick-script.py ${{ github.event.pull_request.number }} branch-3.1    
+
       - name: Auto cherry-pick to branch-4.0
         if: ${{ ((github.event.action == 'labeled' && github.event.label.name == 'dev/4.0.x'))|| ((github.event_name == 'pull_request_target' && github.event.action == 'closed') && contains(github.event.pull_request.labels.*.name, 'dev/4.0.x')) }}
         env:

NOTICE.txt

@@ -1,5 +1,5 @@
 Apache Doris
-Copyright 2018-2025 The Apache Software Foundation
+Copyright 2018-2026 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp

@@ -224,6 +224,12 @@ Status CloudTabletCalcDeleteBitmapTask::handle() const {
     });
     Status status;
     if (_sub_txn_ids.empty()) {
+        // Check empty rowset for non-sub_txn case
+        if (_engine.txn_delete_bitmap_cache().is_empty_rowset(_transaction_id, _tablet_id)) {
+            LOG(INFO) << "tablet=" << _tablet_id << ", txn=" << _transaction_id
+                      << " is empty rowset, skip delete bitmap calculation";
+            return Status::OK();
+        }
         status = _handle_rowset(tablet, _version);
     } else {
         std::stringstream ss;
@@ -237,9 +243,18 @@ Status CloudTabletCalcDeleteBitmapTask::handle() const {
         std::vector<RowsetSharedPtr> invisible_rowsets;
         DeleteBitmapPtr tablet_delete_bitmap =
                 std::make_shared<DeleteBitmap>(tablet->tablet_meta()->delete_bitmap());
-        for (int i = 0; i < _sub_txn_ids.size(); ++i) {
+        size_t empty_rowset_count = 0;
+        for (size_t i = 0; i < _sub_txn_ids.size(); ++i) {
             int64_t sub_txn_id = _sub_txn_ids[i];
             int64_t version = _version + i;
+            // Check empty rowset for each sub_txn using sub_txn_id
+            if (_engine.txn_delete_bitmap_cache().is_empty_rowset(sub_txn_id, _tablet_id)) {
+                LOG(INFO) << "tablet=" << _tablet_id << ", sub_txn=" << sub_txn_id
+                          << ", version=" << version
+                          << " is empty rowset, skip delete bitmap calculation";
+                empty_rowset_count++;
+                continue;
+            }
             LOG(INFO) << "start calc delete bitmap for txn_id=" << _transaction_id
                       << ", sub_txn_id=" << sub_txn_id << ", table_id=" << tablet->table_id()
                       << ", partition_id=" << tablet->partition_id() << ", tablet_id=" << _tablet_id
@@ -254,7 +269,7 @@ Status CloudTabletCalcDeleteBitmapTask::handle() const {
                           << ", cur_version=" << version << ", status=" << status;
                 return status;
             }
-            DCHECK(invisible_rowsets.size() == i + 1);
+            DCHECK(invisible_rowsets.size() == i + 1 - empty_rowset_count);
         }
     }
     DBUG_EXECUTE_IF("CloudCalcDbmTask.handle.return.block",

be/src/cloud/cloud_rowset_builder.cpp

@@ -127,6 +127,15 @@ const RowsetMetaSharedPtr& CloudRowsetBuilder::rowset_meta() {
 
 Status CloudRowsetBuilder::set_txn_related_delete_bitmap() {
     if (_tablet->enable_unique_key_merge_on_write()) {
+        // For empty rowsets when skip_writing_empty_rowset_metadata=true,
+        // store only a lightweight marker instead of full rowset info.
+        // This allows CalcDeleteBitmapTask to detect and skip gracefully,
+        // while using minimal memory (~16 bytes per entry).
+        if (_skip_writing_rowset_metadata) {
+            _engine.txn_delete_bitmap_cache().mark_empty_rowset(_req.txn_id, _tablet->tablet_id(),
+                                                                _req.txn_expiration);
+            return Status::OK();
+        }
         if (config::enable_merge_on_write_correctness_check && _rowset->num_rows() != 0) {
             auto st = _tablet->check_delete_bitmap_correctness(
                     _delete_bitmap, _rowset->end_version() - 1, _req.txn_id, *_rowset_ids);

be/src/cloud/cloud_stream_load_executor.cpp

@@ -104,6 +104,11 @@ Status CloudStreamLoadExecutor::operate_txn_2pc(StreamLoadContext* ctx) {
 
 Status CloudStreamLoadExecutor::commit_txn(StreamLoadContext* ctx) {
     DBUG_EXECUTE_IF("StreamLoadExecutor.commit_txn.block", DBUG_BLOCK);
+    DBUG_EXECUTE_IF("StreamLoadExecutor.commit_txn.crash", {
+        LOG(INFO) << "debug point " << DP_NAME << " trigger crash";
+        volatile int* p = nullptr;
+        *p = 1;
+    });
     // forward to fe to excute commit transaction for MoW table
     if (ctx->is_mow_table() || !config::enable_stream_load_commit_txn_on_be ||
         ctx->load_type == TLoadType::ROUTINE_LOAD) {

be/src/cloud/cloud_tablet_mgr.cpp

@@ -29,6 +29,7 @@
 #include "common/status.h"
 #include "olap/lru_cache.h"
 #include "runtime/memory/cache_policy.h"
+#include "util/debug_points.h"
 #include "util/stack_util.h"
 
 namespace doris {
@@ -148,7 +149,9 @@ CloudTabletMgr::CloudTabletMgr(CloudStorageEngine& engine)
           _tablet_map(std::make_unique<TabletMap>()),
           _cache(std::make_unique<LRUCachePolicy>(
                   CachePolicy::CacheType::CLOUD_TABLET_CACHE, config::tablet_cache_capacity,
-                  LRUCacheType::NUMBER, 0, config::tablet_cache_shards, false /*enable_prune*/)) {}
+                  LRUCacheType::NUMBER, /*sweep time*/ 0, config::tablet_cache_shards,
+                  /*element_count_capacity*/ 0, /*enable_prune*/ false,
+                  /*is_lru_k*/ false)) {}
 
 CloudTabletMgr::~CloudTabletMgr() = default;
 
@@ -163,6 +166,7 @@ Result<std::shared_ptr<CloudTablet>> CloudTabletMgr::get_tablet(int64_t tablet_i
                                                                 SyncRowsetStats* sync_stats,
                                                                 bool force_use_only_cached,
                                                                 bool cache_on_miss) {
+    DBUG_EXECUTE_IF("CloudTabletMgr::get_tablet.block", DBUG_BLOCK);
     // LRU value type. `Value`'s lifetime MUST NOT be longer than `CloudTabletMgr`
     class Value : public LRUCacheValueBase {
     public:

be/src/cloud/cloud_txn_delete_bitmap_cache.cpp

@@ -34,7 +34,9 @@ namespace doris {
 
 CloudTxnDeleteBitmapCache::CloudTxnDeleteBitmapCache(size_t size_in_bytes)
         : LRUCachePolicy(CachePolicy::CacheType::CLOUD_TXN_DELETE_BITMAP_CACHE, size_in_bytes,
-                         LRUCacheType::SIZE, 86400, 4),
+                         LRUCacheType::SIZE, /*stale_sweep_time_s*/ 86400, /*num_shards*/ 4,
+                         /*element_count_capacity*/ 0, /*enable_prune*/ true,
+                         /*is_lru_k*/ false),
           _stop_latch(1) {}
 
 CloudTxnDeleteBitmapCache::~CloudTxnDeleteBitmapCache() {
@@ -229,7 +231,9 @@ void CloudTxnDeleteBitmapCache::remove_expired_tablet_txn_info() {
     std::unique_lock<std::shared_mutex> wlock(_rwlock);
     while (!_expiration_txn.empty()) {
         auto iter = _expiration_txn.begin();
-        if (_txn_map.find(iter->second) == _txn_map.end()) {
+        bool in_txn_map = _txn_map.find(iter->second) != _txn_map.end();
+        bool in_markers = _empty_rowset_markers.find(iter->second) != _empty_rowset_markers.end();
+        if (!in_txn_map && !in_markers) {
             _expiration_txn.erase(iter);
             continue;
         }
@@ -239,6 +243,7 @@ void CloudTxnDeleteBitmapCache::remove_expired_tablet_txn_info() {
         if (iter->first > current_time) {
             break;
         }
+        // Clean from _txn_map if exists
         auto txn_iter = _txn_map.find(iter->second);
         if ((txn_iter != _txn_map.end()) && (iter->first == txn_iter->second.txn_expiration)) {
             LOG_INFO("clean expired delete bitmap")
@@ -251,6 +256,14 @@ void CloudTxnDeleteBitmapCache::remove_expired_tablet_txn_info() {
             erase(cache_key);
             _txn_map.erase(iter->second);
         }
+        // Clean from _empty_rowset_markers if exists
+        auto marker_iter = _empty_rowset_markers.find(iter->second);
+        if (marker_iter != _empty_rowset_markers.end()) {
+            LOG_INFO("clean expired empty rowset marker")
+                    .tag("txn_id", iter->second.txn_id)
+                    .tag("tablet_id", iter->second.tablet_id);
+            _empty_rowset_markers.erase(marker_iter);
+        }
         _expiration_txn.erase(iter);
     }
 }
@@ -272,6 +285,32 @@ void CloudTxnDeleteBitmapCache::remove_unused_tablet_txn_info(TTransactionId tra
     }
 }
 
+void CloudTxnDeleteBitmapCache::mark_empty_rowset(TTransactionId txn_id, int64_t tablet_id,
+                                                  int64_t txn_expiration) {
+    int64_t txn_expiration_min =
+            duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch())
+                    .count() +
+            config::tablet_txn_info_min_expired_seconds;
+    txn_expiration = std::max(txn_expiration_min, txn_expiration);
+
+    if (config::enable_mow_verbose_log) {
+        LOG_INFO("mark empty rowset")
+                .tag("txn_id", txn_id)
+                .tag("tablet_id", tablet_id)
+                .tag("expiration", txn_expiration);
+    }
+    std::unique_lock<std::shared_mutex> wlock(_rwlock);
+    TxnKey txn_key(txn_id, tablet_id);
+    _empty_rowset_markers.emplace(txn_key);
+    _expiration_txn.emplace(txn_expiration, txn_key);
+}
+
+bool CloudTxnDeleteBitmapCache::is_empty_rowset(TTransactionId txn_id, int64_t tablet_id) {
+    std::shared_lock<std::shared_mutex> rlock(_rwlock);
+    TxnKey txn_key(txn_id, tablet_id);
+    return _empty_rowset_markers.contains(txn_key);
+}
+
 void CloudTxnDeleteBitmapCache::_clean_thread_callback() {
     do {
         remove_expired_tablet_txn_info();

be/src/cloud/cloud_txn_delete_bitmap_cache.h

@@ -59,6 +59,16 @@ class CloudTxnDeleteBitmapCache : public LRUCachePolicy {
 
     void remove_unused_tablet_txn_info(TTransactionId transaction_id, int64_t tablet_id);
 
+    // Mark a rowset as empty/skipped (lightweight marker, no rowset stored)
+    // Used for empty rowsets when skip_writing_empty_rowset_metadata is enabled
+    void mark_empty_rowset(TTransactionId txn_id, int64_t tablet_id, int64_t txn_expiration);
+
+    // Check if this is a known empty/skipped rowset
+    // Returns true if was marked as empty rowset
+    // Note: Does not remove the marker, as CalcDeleteBitmapTask may retry.
+    // Cleanup is handled by expiration-based removal in remove_expired_tablet_txn_info()
+    bool is_empty_rowset(TTransactionId txn_id, int64_t tablet_id);
+
     // !!!ATTENTION!!!: the delete bitmap stored in CloudTxnDeleteBitmapCache contains sentinel marks,
     // and the version in BitmapKey is DeleteBitmap::TEMP_VERSION_COMMON.
     // when using delete bitmap from this cache, the caller should manually remove these marks if don't need it
@@ -107,6 +117,9 @@ class CloudTxnDeleteBitmapCache : public LRUCachePolicy {
 
     std::map<TxnKey, TxnVal> _txn_map;
     std::multimap<int64_t, TxnKey> _expiration_txn;
+    // Lightweight markers for empty/skipped rowsets (only stores TxnKey, ~16 bytes per entry)
+    // Used to track empty rowsets that were not committed to meta-service
+    std::set<TxnKey> _empty_rowset_markers;
     std::shared_mutex _rwlock;
     std::shared_ptr<Thread> _clean_thread;
     CountDownLatch _stop_latch;

be/src/cloud/config.cpp

@@ -110,7 +110,7 @@ DEFINE_mBool(enable_cloud_tablet_report, "true");
 
 DEFINE_mInt32(delete_bitmap_rpc_retry_times, "25");
 
-DEFINE_mInt64(meta_service_rpc_reconnect_interval_ms, "5000");
+DEFINE_mInt64(meta_service_rpc_reconnect_interval_ms, "100");
 
 DEFINE_mInt32(meta_service_conflict_error_retry_times, "10");
 
@@ -135,7 +135,7 @@ DEFINE_mInt64(warm_up_rowset_sync_wait_max_timeout_ms, "120000");
 DEFINE_mBool(enable_warmup_immediately_on_new_rowset, "false");
 
 // Packed file manager config
-DEFINE_mBool(enable_packed_file, "true");
+DEFINE_mBool(enable_packed_file, "false");
 DEFINE_mInt64(packed_file_size_threshold_bytes, "5242880"); // 5MB
 DEFINE_mInt64(packed_file_time_threshold_ms, "100");        // 100ms
 DEFINE_mInt64(packed_file_try_lock_timeout_ms, "5");        // 5ms

be/src/cloud/injection_point_action.cpp

@@ -279,6 +279,17 @@ void set_return_error(const std::string& point, HttpRequest* req) {
     HttpChannel::send_reply(req, HttpStatus::OK, "OK");
 }
 
+void set_segfault(const std::string& point, HttpRequest* req) {
+    auto sp = SyncPoint::get_instance();
+    sp->set_call_back(point, [point](auto&&) {
+        LOG(INFO) << "injection point hit, point=" << point << " trigger segfault";
+        // Intentional null dereference to crash the BE for testing.
+        volatile int* p = nullptr;
+        *p = 1;
+    });
+    HttpChannel::send_reply(req, HttpStatus::OK, "OK");
+}
+
 void handle_set(HttpRequest* req) {
     auto& point = req->param("name");
     if (point.empty()) {
@@ -302,6 +313,9 @@ void handle_set(HttpRequest* req) {
     } else if (behavior == "return_error") {
         set_return_error(point, req);
         return;
+    } else if (behavior == "segfault") {
+        set_segfault(point, req);
+        return;
     }
     HttpChannel::send_reply(req, HttpStatus::BAD_REQUEST, "unknown behavior: " + behavior);
 }
@@ -377,13 +391,15 @@ InjectionPointAction::InjectionPointAction() = default;
 //                 which is an int, valid values can be found in status.h, e.g. -235 or -230,
 //                 if `code` is not present return Status::InternalError. Optional `probability`
 //                 determines the percentage of times to inject the error (default 100).
+// * segfault: dereference a null pointer to crash BE intentionally
 // ```
 // curl "be_ip:http_port/api/injection_point/set?name=${injection_point_name}&behavior=sleep&duration=${x_millsec}" # sleep x millisecs
 // curl "be_ip:http_port/api/injection_point/set?name=${injection_point_name}&behavior=return" # return void
 // curl "be_ip:http_port/api/injection_point/set?name=${injection_point_name}&behavior=return_ok" # return ok
 // curl "be_ip:http_port/api/injection_point/set?name=${injection_point_name}&behavior=return_error" # internal error
 // curl "be_ip:http_port/api/injection_point/set?name=${injection_point_name}&behavior=return_error&code=${code}" # -235
 // curl "be_ip:http_port/api/injection_point/set?name=${injection_point_name}&behavior=return_error&code=${code}&probability=50" # inject with 50% probability
+// curl "be_ip:http_port/api/injection_point/set?name=${injection_point_name}&behavior=segfault" # crash BE
 // ```
 void InjectionPointAction::handle(HttpRequest* req) {
     LOG(INFO) << "handle InjectionPointAction " << req->debug_string();