diff --git a/be/src/io/fs/packed_file_manager.cpp b/be/src/io/fs/packed_file_manager.cpp
index 377f4cce67e355..cc215016aecd6c 100644
--- a/be/src/io/fs/packed_file_manager.cpp
+++ b/be/src/io/fs/packed_file_manager.cpp
@@ -256,6 +256,7 @@ Status PackedFileManager::append_small_file(const std::string& path, const Slice
     location.packed_file_path = active_state->packed_file_path;
     location.offset = active_state->current_offset;
     location.size = data.get_size();
+    location.create_time = std::time(nullptr);
     location.tablet_id = info.tablet_id;
     location.rowset_id = info.rowset_id;
     location.resource_id = info.resource_id;
@@ -809,30 +810,12 @@ void PackedFileManager::cleanup_expired_data() {
 
     // Clean up expired global index entries
     {
-        std::unordered_set<std::string> active_packed_files;
-        {
-            std::lock_guard<std::timed_mutex> current_lock(_current_packed_file_mutex);
-            for (const auto& [resource_id, state] : _current_packed_files) {
-                if (state) {
-                    active_packed_files.insert(state->packed_file_path);
-                }
-            }
-        }
-        {
-            std::lock_guard<std::mutex> merge_lock(_packed_files_mutex);
-            for (const auto& [path, state] : _uploading_packed_files) {
-                active_packed_files.insert(path);
-            }
-            for (const auto& [path, state] : _uploaded_packed_files) {
-                active_packed_files.insert(path);
-            }
-        }
-
         std::lock_guard<std::mutex> global_lock(_global_index_mutex);
         auto it = _global_slice_locations.begin();
         while (it != _global_slice_locations.end()) {
             const auto& index = it->second;
-            if (active_packed_files.find(index.packed_file_path) == active_packed_files.end()) {
+            if (index.create_time > 0 &&
+                current_time - index.create_time > config::uploaded_file_retention_seconds) {
                 it = _global_slice_locations.erase(it);
             } else {
                 ++it;
diff --git a/be/src/io/fs/packed_file_manager.h b/be/src/io/fs/packed_file_manager.h
index 5957ce15e8af96..d1415ec302fbb6 100644
--- a/be/src/io/fs/packed_file_manager.h
+++ b/be/src/io/fs/packed_file_manager.h
@@ -45,6 +45,7 @@ struct PackedSliceLocation {
     std::string packed_file_path;
     int64_t offset;
     int64_t size;
+    int64_t create_time = 0;
     int64_t tablet_id = 0;
     std::string rowset_id;
     std::string resource_id;