Skip to content

Commit 1989fa5

Browse files
authored
chore(cubestore): Improve remoteFs metrics (#6553)
1 parent 1296ddb commit 1989fa5

File tree

2 files changed

+16
-11
lines changed

2 files changed

+16
-11
lines changed

rust/cubestore/cubestore/src/remotefs/gcs.rs

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -254,13 +254,6 @@ impl RemoteFs for GCSRemoteFs {
254254
}
255255

256256
async fn list_with_metadata(&self, remote_prefix: &str) -> Result<Vec<RemoteFile>, CubeError> {
257-
app_metrics::REMOTE_FS_OPERATION_CORE.add_with_tags(
258-
1,
259-
Some(&vec![
260-
"operation:list".to_string(),
261-
"driver:gcs".to_string(),
262-
]),
263-
);
264257
let prefix = self.gcs_path(remote_prefix);
265258
let list = Object::list_prefix(self.bucket.as_str(), prefix.as_str()).await?;
266259
let leading_slash = Regex::new(format!("^{}", self.gcs_path("")).as_str()).unwrap();
@@ -281,6 +274,17 @@ impl RemoteFs for GCSRemoteFs {
281274
.flatten()
282275
.flatten()
283276
.collect::<Vec<_>>();
277+
let mut pages_count = result.len() / 1_000;
278+
if result.len() % 1_000 > 0 {
279+
pages_count += 1;
280+
}
281+
app_metrics::REMOTE_FS_OPERATION_CORE.add_with_tags(
282+
pages_count as i64,
283+
Some(&vec![
284+
"operation:list".to_string(),
285+
"driver:gcs".to_string(),
286+
]),
287+
);
284288
Ok(result)
285289
}
286290

rust/cubestore/cubestore/src/remotefs/s3.rs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -274,13 +274,14 @@ impl RemoteFs for S3RemoteFs {
274274
}
275275

276276
async fn list_with_metadata(&self, remote_prefix: &str) -> Result<Vec<RemoteFile>, CubeError> {
277-
app_metrics::REMOTE_FS_OPERATION_CORE.add_with_tags(
278-
1,
279-
Some(&vec!["operation:list".to_string(), "driver:s3".to_string()]),
280-
);
281277
let path = self.s3_path(remote_prefix);
282278
let bucket = self.bucket.read().unwrap().clone();
283279
let list = cube_ext::spawn_blocking(move || bucket.list_blocking(path, None)).await??;
280+
let pages_count = list.len();
281+
app_metrics::REMOTE_FS_OPERATION_CORE.add_with_tags(
282+
pages_count as i64,
283+
Some(&vec!["operation:list".to_string(), "driver:s3".to_string()]),
284+
);
284285
let leading_slash = Regex::new(format!("^{}", self.s3_path("")).as_str()).unwrap();
285286
let result = list
286287
.iter()

0 commit comments

Comments
 (0)