Perform self-profile compression in a separate thread

Author: Kobzol

collector/src/bin/collector.rs

@@ -1816,7 +1816,9 @@ fn build_async_runtime() -> Runtime {
     // We want to minimize noise from the runtime
     builder
         .worker_threads(1)
-        .max_blocking_threads(1)
+        .max_blocking_threads(8)
+        // Do not keep blocking threads alive for long
+        .thread_keep_alive(Duration::from_secs(1))
         .enable_time()
         .enable_io();
     builder.build().expect("built runtime")

collector/src/self_profile.rs

@@ -202,10 +202,19 @@ impl SelfProfileStorage for S3SelfProfileStorage {
                         data.len()
                     );
                     let start = Instant::now();
-                    let mut data = snap::read::FrameEncoder::new(&data[..]);
-                    let mut compressed = Vec::new();
-                    data.read_to_end(&mut compressed)
-                        .context("cannot compress self-profile data")?;
+
+                    // This is synchronous and blocks the event loop, so we should do it on a
+                    // worker thread
+                    let compressed = tokio::task::spawn_blocking(move || {
+                        let mut data = snap::read::FrameEncoder::new(&data[..]);
+                        let mut compressed = Vec::new();
+
+                        data.read_to_end(&mut compressed)
+                            .context("cannot compress self-profile data")?;
+                        anyhow::Ok(compressed)
+                    })
+                    .await??;
+
                     log::trace!(
                         "Compress self-profile duration: {}, size: {}",
                         start.elapsed().as_secs_f64(),