Merge remote-tracking branch 'origin/feature/Enhancement-dynamic_partition' into feature/Enhancement-dynamic_partition

Author: hacklu-tu

.gitattributes

@@ -14,6 +14,6 @@
 *.thrift text eol=lf
 *.proto text eol=lf
 *.conf text eol=lf
-*.out text eol=lf -diff
+*.out text eol=lf diff
 *.groovy -linguist-detectable
 regression-test/ export-ignore

.github/workflows/auto-cherry-pick.yml

@@ -31,7 +31,7 @@ permissions:
 jobs:
   auto_cherry_pick:
     runs-on: ubuntu-latest
-    if: ${{(contains(github.event.pull_request.labels.*.name, 'dev/3.0.x') || contains(github.event.pull_request.labels.*.name, 'dev/2.1.x') || contains(github.event.pull_request.labels.*.name, 'dev/3.1.x') || github.event.label.name == 'dev/3.0.x' || github.event.label.name == 'dev/2.1.x' || github.event.label.name == 'dev/3.1.x') && github.event.pull_request.merged == true }}
+    if: ${{(contains(github.event.pull_request.labels.*.name, 'dev/3.0.x') || contains(github.event.pull_request.labels.*.name, 'dev/4.0.x') || contains(github.event.pull_request.labels.*.name, 'dev/2.1.x') || contains(github.event.pull_request.labels.*.name, 'dev/3.1.x') || github.event.label.name == 'dev/4.0.x' || github.event.label.name == 'dev/3.0.x' || github.event.label.name == 'dev/2.1.x' || github.event.label.name == 'dev/3.1.x') && github.event.pull_request.merged == true }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
@@ -77,4 +77,12 @@ jobs:
           REPO_NAME: ${{ github.repository }}
           CONFLICT_LABEL: dev/3.1.x-conflict
         run: |
-            python tools/auto-pick-script.py ${{ github.event.pull_request.number }} branch-3.1         
+            python tools/auto-pick-script.py ${{ github.event.pull_request.number }} branch-3.1    
+      - name: Auto cherry-pick to branch-4.0
+        if: ${{ ((github.event.action == 'labeled' && github.event.label.name == 'dev/4.0.x'))|| ((github.event_name == 'pull_request_target' && github.event.action == 'closed') && contains(github.event.pull_request.labels.*.name, 'dev/4.0.x')) }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO_NAME: ${{ github.repository }}
+          CONFLICT_LABEL: dev/4.0.x-conflict
+        run: |
+          python tools/auto-pick-script.py ${{ github.event.pull_request.number }} branch-4.0             

.github/workflows/comment-to-trigger-teamcity.yml

@@ -59,7 +59,6 @@ jobs:
             "${COMMENT_BODY}" == *'run vault_p0'* ||
             "${COMMENT_BODY}" == *'run nonConcurrent'* ||
             "${COMMENT_BODY}" == *'run check_coverage'* ||
-            "${COMMENT_BODY}" == *'run arm'* ||
             "${COMMENT_BODY}" == *'run performance'* ]]; then
             echo "comment_trigger=true" | tee -a "$GITHUB_OUTPUT"
             echo "comment_skip=false" | tee -a "$GITHUB_OUTPUT"
@@ -83,6 +82,10 @@ jobs:
                 echo "COMMENT_USER_ID ${COMMENT_USER_ID} is allowed to skip buildall for branch-3.1"
                 echo "comment_trigger=false" | tee -a "$GITHUB_OUTPUT"
                 echo "comment_skip=true" | tee -a "$GITHUB_OUTPUT"
+            elif [[ "${COMMENT_USER_ID}" == '9208457' && "${TARGET_BRANCH}" == *'branch-4.0'* ]]; then
+                echo "COMMENT_USER_ID ${COMMENT_USER_ID} is allowed to skip buildall for branch-4.0"
+                echo "comment_trigger=false" | tee -a "$GITHUB_OUTPUT"
+                echo "comment_skip=true" | tee -a "$GITHUB_OUTPUT"
             else
                 echo "COMMENT_USER_ID ${COMMENT_USER_ID} is not allowed to skip buildall."
                 exit
@@ -117,7 +120,7 @@ jobs:
         echo "TARGET_BRANCH='${TARGET_BRANCH}'" | tee -a "$GITHUB_OUTPUT"
         echo "COMMENT_BODY='${COMMENT_BODY}'" | tee -a "$GITHUB_OUTPUT"
 
-        reg="run (buildall|compile|p0|p1|feut|beut|cloudut|external|clickbench|cloud_p0|cloud_p1|vault_p0|nonConcurrent|arm|performance|check_coverage)( [1-9]*[0-9]+)*"
+        reg="run (buildall|compile|p0|p1|feut|beut|cloudut|external|clickbench|cloud_p0|cloud_p1|vault_p0|nonConcurrent|performance|check_coverage)( [1-9]*[0-9]+)*"
         COMMENT_TRIGGER_TYPE="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p | sed 's/\r//g')"
         COMMENT_REPEAT_TIMES="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p | sed 's/\r//g')"
         echo "COMMENT_TRIGGER_TYPE=${COMMENT_TRIGGER_TYPE}" | tee -a "$GITHUB_OUTPUT"
@@ -152,12 +155,10 @@ jobs:
           if file_changed_regression_p0; then
             echo "changed_p0=true" | tee -a "$GITHUB_OUTPUT"
             echo "changed_external=true" | tee -a "$GITHUB_OUTPUT"
-            echo "changed_arm=true" | tee -a "$GITHUB_OUTPUT"
             echo "changed_nonConcurrent=true" | tee -a "$GITHUB_OUTPUT"
           else
             echo "changed_p0=false" | tee -a "$GITHUB_OUTPUT"
             echo "changed_external=false" | tee -a "$GITHUB_OUTPUT"
-            echo "changed_arm=false" | tee -a "$GITHUB_OUTPUT"
             echo "changed_nonConcurrent=false" | tee -a "$GITHUB_OUTPUT"
           fi
           if file_changed_regression_p1; then
@@ -194,7 +195,6 @@ jobs:
           echo "changed_cloud_ut=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_p0=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_external=true" | tee -a "$GITHUB_OUTPUT"
-          echo "changed_arm=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_p1=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_performance=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_cloud_p0=true" | tee -a "$GITHUB_OUTPUT"
@@ -403,19 +403,6 @@ jobs:
           "cloud_p1" \
           "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
 
-    - name: "Trigger or Skip arm"
-      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["arm", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
-      run: |
-        source ./regression-test/pipeline/common/teamcity-utils.sh
-        set -x
-        trigger_or_skip_build \
-          "${{ steps.changes.outputs.changed_arm }}" \
-          "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
-          "${{ steps.parse.outputs.TARGET_BRANCH }}" \
-          "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
-          "arm" \
-          "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
-
     - name: "Trigger or Skip performance"
       if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["performance", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
       run: |
@@ -440,7 +427,6 @@ jobs:
         skip_build "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" p1
         skip_build "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" external
         skip_build "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" performance
-        skip_build "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" arm
         skip_build "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" cloud_p0
         skip_build "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" cloud_p1
         skip_build "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" cloudut

be/src/agent/task_worker_pool.cpp

@@ -895,7 +895,7 @@ void update_tablet_meta_callback(StorageEngine& engine, const TAgentTaskRequest&
             tablet->tablet_meta()->mutable_tablet_schema()->set_is_in_memory(
                     tablet_meta_info.is_in_memory);
             std::shared_lock rlock(tablet->get_header_lock());
-            for (auto& rowset_meta : tablet->tablet_meta()->all_mutable_rs_metas()) {
+            for (auto& [_, rowset_meta] : tablet->tablet_meta()->all_mutable_rs_metas()) {
                 rowset_meta->tablet_schema()->set_is_in_memory(tablet_meta_info.is_in_memory);
             }
             tablet->tablet_schema_unlocked()->set_is_in_memory(tablet_meta_info.is_in_memory);
@@ -993,7 +993,7 @@ void update_tablet_meta_callback(StorageEngine& engine, const TAgentTaskRequest&
             std::shared_lock rlock(tablet->get_header_lock());
             tablet->tablet_meta()->mutable_tablet_schema()->set_enable_single_replica_compaction(
                     tablet_meta_info.enable_single_replica_compaction);
-            for (auto& rowset_meta : tablet->tablet_meta()->all_mutable_rs_metas()) {
+            for (auto& [_, rowset_meta] : tablet->tablet_meta()->all_mutable_rs_metas()) {
                 rowset_meta->tablet_schema()->set_enable_single_replica_compaction(
                         tablet_meta_info.enable_single_replica_compaction);
             }
@@ -1005,7 +1005,7 @@ void update_tablet_meta_callback(StorageEngine& engine, const TAgentTaskRequest&
             std::shared_lock rlock(tablet->get_header_lock());
             tablet->tablet_meta()->mutable_tablet_schema()->set_disable_auto_compaction(
                     tablet_meta_info.disable_auto_compaction);
-            for (auto& rowset_meta : tablet->tablet_meta()->all_mutable_rs_metas()) {
+            for (auto& [_, rowset_meta] : tablet->tablet_meta()->all_mutable_rs_metas()) {
                 rowset_meta->tablet_schema()->set_disable_auto_compaction(
                         tablet_meta_info.disable_auto_compaction);
             }
@@ -1018,7 +1018,7 @@ void update_tablet_meta_callback(StorageEngine& engine, const TAgentTaskRequest&
             std::shared_lock rlock(tablet->get_header_lock());
             tablet->tablet_meta()->mutable_tablet_schema()->set_skip_write_index_on_load(
                     tablet_meta_info.skip_write_index_on_load);
-            for (auto& rowset_meta : tablet->tablet_meta()->all_mutable_rs_metas()) {
+            for (auto& [_, rowset_meta] : tablet->tablet_meta()->all_mutable_rs_metas()) {
                 rowset_meta->tablet_schema()->set_skip_write_index_on_load(
                         tablet_meta_info.skip_write_index_on_load);
             }
@@ -1093,6 +1093,7 @@ void report_task_callback(const ClusterInfo* cluster_info) {
         }
     }
     request.__set_backend(BackendOptions::get_local_backend());
+    request.__set_running_tasks(ExecEnv::GetInstance()->fragment_mgr()->running_query_num());
     bool succ = handle_report(request, cluster_info, "task");
     report_task_total << 1;
     if (!succ) [[unlikely]] {

be/src/cloud/cloud_meta_mgr.cpp

@@ -351,7 +351,8 @@ static std::string debug_info(const Request& req) {
         return fmt::format(" table_id={}, lock_id={}", req.table_id(), req.lock_id());
     } else if constexpr (is_any_v<Request, GetTabletRequest>) {
         return fmt::format(" tablet_id={}", req.tablet_id());
-    } else if constexpr (is_any_v<Request, GetObjStoreInfoRequest>) {
+    } else if constexpr (is_any_v<Request, GetObjStoreInfoRequest, ListSnapshotRequest,
+                                  GetInstanceRequest>) {
         return "";
     } else if constexpr (is_any_v<Request, CreateRowsetRequest>) {
         return fmt::format(" tablet_id={}", req.rowset_meta().tablet_id());
@@ -2034,7 +2035,7 @@ Status CloudMetaMgr::fill_version_holes(CloudTablet* tablet, int64_t max_version
     }
 
     Versions existing_versions;
-    for (const auto& rs : tablet->tablet_meta()->all_rs_metas()) {
+    for (const auto& [_, rs] : tablet->tablet_meta()->all_rs_metas()) {
         existing_versions.emplace_back(rs->version());
     }
 
@@ -2176,5 +2177,36 @@ Status CloudMetaMgr::create_empty_rowset_for_hole(CloudTablet* tablet, int64_t v
 
     return Status::OK();
 }
+
+Status CloudMetaMgr::list_snapshot(std::vector<SnapshotInfoPB>& snapshots) {
+    ListSnapshotRequest req;
+    ListSnapshotResponse res;
+    req.set_cloud_unique_id(config::cloud_unique_id);
+    req.set_include_aborted(true);
+    RETURN_IF_ERROR(retry_rpc("list snapshot", req, &res, &MetaService_Stub::list_snapshot));
+    for (auto& snapshot : snapshots) {
+        snapshots.emplace_back(snapshot);
+    }
+    return Status::OK();
+}
+
+Status CloudMetaMgr::get_snapshot_properties(SnapshotSwitchStatus& switch_status,
+                                             int64_t& max_reserved_snapshots,
+                                             int64_t& snapshot_interval_seconds) {
+    GetInstanceRequest req;
+    GetInstanceResponse res;
+    req.set_cloud_unique_id(config::cloud_unique_id);
+    RETURN_IF_ERROR(
+            retry_rpc("get snapshot properties", req, &res, &MetaService_Stub::get_instance));
+    switch_status = res.instance().has_snapshot_switch_status()
+                            ? res.instance().snapshot_switch_status()
+                            : SnapshotSwitchStatus::SNAPSHOT_SWITCH_DISABLED;
+    max_reserved_snapshots =
+            res.instance().has_max_reserved_snapshot() ? res.instance().max_reserved_snapshot() : 0;
+    snapshot_interval_seconds = res.instance().has_snapshot_interval_seconds()
+                                        ? res.instance().snapshot_interval_seconds()
+                                        : 3600;
+    return Status::OK();
+}
 #include "common/compile_check_end.h"
 } // namespace doris::cloud

be/src/cloud/cloud_meta_mgr.h

@@ -161,6 +161,11 @@ class CloudMetaMgr {
                                         RowsetMetaSharedPtr prev_rowset_meta,
                                         RowsetSharedPtr* rowset);
 
+    Status list_snapshot(std::vector<SnapshotInfoPB>& snapshots);
+    Status get_snapshot_properties(SnapshotSwitchStatus& switch_status,
+                                   int64_t& max_reserved_snapshots,
+                                   int64_t& snapshot_interval_seconds);
+
 private:
     bool sync_tablet_delete_bitmap_by_cache(CloudTablet* tablet, int64_t old_max_version,
                                             std::ranges::range auto&& rs_metas,

be/src/cloud/cloud_schema_change_job.cpp

@@ -490,12 +490,14 @@ Status CloudSchemaChangeJob::_process_delete_bitmap(int64_t alter_version,
     tmp_meta->delete_bitmap().delete_bitmap.clear();
     // Keep only version [0-1] rowset, other rowsets will be added in _output_rowsets
     auto& rs_metas = tmp_meta->all_mutable_rs_metas();
-    rs_metas.erase(std::remove_if(rs_metas.begin(), rs_metas.end(),
-                                  [](const RowsetMetaSharedPtr& rs_meta) {
-                                      return !(rs_meta->version().first == 0 &&
-                                               rs_meta->version().second == 1);
-                                  }),
-                   rs_metas.end());
+    for (auto it = rs_metas.begin(); it != rs_metas.end();) {
+        const auto& rs_meta = it->second;
+        if (rs_meta->version().first == 0 && rs_meta->version().second == 1) {
+            ++it;
+        } else {
+            it = rs_metas.erase(it);
+        }
+    }
 
     std::shared_ptr<CloudTablet> tmp_tablet =
             std::make_shared<CloudTablet>(_cloud_storage_engine, tmp_meta);

be/src/cloud/cloud_snapshot_mgr.cpp

@@ -83,7 +83,7 @@ Status CloudSnapshotMgr::make_snapshot(int64_t target_tablet_id, StorageResource
         if (tablet_meta.all_rs_metas().size() > 0) {
             tablet_meta_pb.mutable_inc_rs_metas()->Reserve(
                     cast_set<int>(tablet_meta.all_rs_metas().size()));
-            for (auto& rs : tablet_meta.all_rs_metas()) {
+            for (auto& [_, rs] : tablet_meta.all_rs_metas()) {
                 rs->to_rowset_pb(tablet_meta_pb.add_rs_metas());
             }
         }

be/src/cloud/cloud_tablet.cpp

@@ -297,10 +297,11 @@ Status CloudTablet::capture_rs_readers_with_freshness_tolerance(
         return ret;
     };
     // use std::views::concat after C++26
-    bool should_fallback = std::ranges::any_of(_tablet_meta->all_rs_metas(),
-                                               should_be_visible_but_not_warmed_up) ||
-                           std::ranges::any_of(_tablet_meta->all_stale_rs_metas(),
-                                               should_be_visible_but_not_warmed_up);
+    bool should_fallback =
+            std::ranges::any_of(std::views::values(_tablet_meta->all_rs_metas()),
+                                should_be_visible_but_not_warmed_up) ||
+            std::ranges::any_of(std::views::values(_tablet_meta->all_stale_rs_metas()),
+                                should_be_visible_but_not_warmed_up);
     if (should_fallback) {
         rlock.unlock();
         g_capture_with_freshness_tolerance_fallback_count << 1;
@@ -983,7 +984,7 @@ int64_t CloudTablet::get_cloud_base_compaction_score() const {
         bool has_delete = false;
         int64_t point = cumulative_layer_point();
         std::shared_lock<std::shared_mutex> rlock(_meta_lock);
-        for (const auto& rs_meta : _tablet_meta->all_rs_metas()) {
+        for (const auto& [_, rs_meta] : _tablet_meta->all_rs_metas()) {
             if (rs_meta->start_version() >= point) {
                 continue;
             }