Merge pull request #2393 from Kobzol/self-profile-measure

Perform self-profile compression in a separate thread

Author: Kobzol

collector/src/bin/collector.rs

@@ -1816,7 +1816,9 @@ fn build_async_runtime() -> Runtime {
     // We want to minimize noise from the runtime
     builder
         .worker_threads(1)
-        .max_blocking_threads(1)
+        .max_blocking_threads(8)
+        // Do not keep blocking threads alive for long
+        .thread_keep_alive(Duration::from_secs(1))
         .enable_time()
         .enable_io();
     builder.build().expect("built runtime")

collector/src/self_profile.rs

@@ -192,21 +192,49 @@ impl SelfProfileStorage for S3SelfProfileStorage {
             let upload = tempfile::NamedTempFile::new().context("cannot create temporary file")?;
             match files {
                 SelfProfileFiles::Eight { file } => {
+                    let start = Instant::now();
                     let data = tokio::fs::read(file)
                         .await
                         .context("cannot read self-profile data")?;
-                    let mut data = snap::read::FrameEncoder::new(&data[..]);
-                    let mut compressed = Vec::new();
-                    data.read_to_end(&mut compressed)
-                        .context("cannot compress self-profile data")?;
+                    log::trace!(
+                        "Read self-profile duration: {}, size: {}",
+                        start.elapsed().as_secs_f64(),
+                        data.len()
+                    );
+                    let start = Instant::now();
+
+                    // This is synchronous and blocks the event loop, so we should do it on a
+                    // worker thread
+                    let compressed = tokio::task::spawn_blocking(move || {
+                        let mut data = snap::read::FrameEncoder::new(&data[..]);
+                        let mut compressed = Vec::new();
+
+                        data.read_to_end(&mut compressed)
+                            .context("cannot compress self-profile data")?;
+                        anyhow::Ok(compressed)
+                    })
+                    .await??;
+
+                    log::trace!(
+                        "Compress self-profile duration: {}, size: {}",
+                        start.elapsed().as_secs_f64(),
+                        compressed.len()
+                    );
+                    let start = Instant::now();
                     tokio::fs::write(upload.path(), &compressed)
                         .await
                         .context("cannot write compressed self-profile data")?;
+                    log::trace!(
+                        "Write self-profile duration: {}",
+                        start.elapsed().as_secs_f64()
+                    );
+                    compressed
                 }
             };
 
             log::info!("Uploading self-profile to {}", file_path.display());
 
+            let start = Instant::now();
             let output = tokio::process::Command::new("aws")
                 .arg("s3")
                 .arg("cp")
@@ -232,6 +260,10 @@ impl SelfProfileStorage for S3SelfProfileStorage {
                     String::from_utf8_lossy(&output.stderr)
                 ));
             }
+            log::trace!(
+                "Upload self-profile duration: {}",
+                start.elapsed().as_secs_f64()
+            );
             Ok(())
         })
     }