fix: reset stream stats with limit (openobserve#6992)

Author: hengfeiyang

src/cli/basic/cli.rs

@@ -327,9 +327,20 @@ pub async fn cli() -> Result<bool, anyhow::Error> {
                     // load stream list
                     db::schema::cache().await?;
                     // update stats from file list
-                    compact::stats::update_stats_from_file_list()
-                        .await
-                        .expect("file list remote calculate stats failed");
+                    loop {
+                        let ret = compact::stats::update_stats_from_file_list()
+                            .await
+                            .expect("file list remote calculate stats failed");
+                        let Some(offset) = ret else {
+                            break;
+                        };
+                        log::info!(
+                            "keep updating stats from file list, offset: {:?} ...",
+                            offset
+                        );
+                        tokio::time::sleep(tokio::time::Duration::from_millis(50)).await;
+                    }
+                    log::info!("update stats from file list success");
                 }
                 _ => {
                     return Err(anyhow::anyhow!(

src/service/compact/stats.rs

@@ -50,6 +50,11 @@ pub async fn update_stats_from_file_list() -> Result<Option<(i64, i64)>, anyhow:
         Some((offset, latest_pk))
     };
 
+    // there is no new data to process
+    if offset == latest_pk {
+        return Ok(None);
+    }
+
     // get stats from file_list
     let orgs = db::schema::list_organizations_from_cache().await;
     for org_id in orgs {