Turbopack: allow gzip compression on trace files

Cargo.toml

@@ -367,6 +367,7 @@ dhat = { version = "0.3.2" }
 dunce = "1.0.3"
 either = "1.9.0"
 erased-serde = "0.4.5"
+flate2 = "1.0.28"
 futures = "0.3.31"
 futures-util = "0.3.31"
 futures-retry = "0.6.0"

crates/napi/Cargo.toml

@@ -55,10 +55,11 @@ ignored = [
 ]
 
 [dependencies]
-anyhow = "1.0.66"
+anyhow = { workspace = true }
 console-subscriber = { workspace = true, optional = true }
 dhat = { workspace = true, optional = true }
 either = { workspace = true }
+flate2 = { workspace = true }
 futures-util = { workspace = true }
 owo-colors = { workspace = true }
 napi = { workspace = true }

crates/napi/src/lib.rs

@@ -30,6 +30,7 @@ DEALINGS IN THE SOFTWARE.
 //#![deny(clippy::all)]
 #![feature(arbitrary_self_types)]
 #![feature(arbitrary_self_types_pointers)]
+#![feature(iter_intersperse)]
 
 #[macro_use]
 extern crate napi_derive;

crates/napi/src/next_api/project.rs

@@ -1,6 +1,7 @@
 use std::{borrow::Cow, io::Write, path::PathBuf, sync::Arc, thread, time::Duration};
 
 use anyhow::{Context, Result, anyhow, bail};
+use flate2::write::GzEncoder;
 use futures_util::TryFutureExt;
 use napi::{
     Env, JsFunction, JsObject, Status,
@@ -363,21 +364,33 @@ pub fn project_new(
     }
 
     if let Some(mut trace) = trace {
-        // Trace presets
-        match trace.as_str() {
-            "overview" | "1" => {
-                trace = TRACING_NEXT_OVERVIEW_TARGETS.join(",");
-            }
-            "next" => {
-                trace = TRACING_NEXT_TARGETS.join(",");
-            }
-            "turbopack" => {
-                trace = TRACING_NEXT_TURBOPACK_TARGETS.join(",");
-            }
-            "turbo-tasks" => {
-                trace = TRACING_NEXT_TURBO_TASKS_TARGETS.join(",");
-            }
-            _ => {}
+        trace = trace
+            .split(",")
+            .map(|item| {
+                // Trace presets
+                match item {
+                    "overview" | "1" => Cow::Owned(TRACING_NEXT_OVERVIEW_TARGETS.join(",")),
+                    "next" => Cow::Owned(TRACING_NEXT_TARGETS.join(",")),
+                    "turbopack" => Cow::Owned(TRACING_NEXT_TURBOPACK_TARGETS.join(",")),
+                    "turbo-tasks" => Cow::Owned(TRACING_NEXT_TURBO_TASKS_TARGETS.join(",")),
+                    _ => Cow::Borrowed(item),
+                }
+            })
+            .intersperse_with(|| Cow::Borrowed(","))
+            .collect::<String>();
+
+        enum Compression {
+            None,
+            GzipFast,
+            GzipBest,
+        }
+        let mut compress = Compression::None;
+        if trace.ends_with(",gz") {
+            trace.drain(trace.len() - 3..);
+            compress = Compression::GzipFast;
+        } else if trace.ends_with(",gz-best") {
+            trace.drain(trace.len() - 8..);
+            compress = Compression::GzipBest;
         }
 
         let subscriber = Registry::default();
@@ -396,9 +409,26 @@ pub fn project_new(
         std::fs::create_dir_all(&internal_dir)
             .context("Unable to create .next directory")
             .unwrap();
-        let trace_file = internal_dir.join("trace-turbopack");
-        let trace_writer = std::fs::File::create(trace_file.clone()).unwrap();
-        let (trace_writer, trace_writer_guard) = TraceWriter::new(trace_writer);
+        let trace_file;
+        let (trace_writer, trace_writer_guard) = match compress {
+            Compression::None => {
+                trace_file = internal_dir.join("trace-turbopack");
+                let trace_writer = std::fs::File::create(trace_file.clone()).unwrap();
+                TraceWriter::new(trace_writer)
+            }
+            Compression::GzipFast => {
+                trace_file = internal_dir.join("trace-turbopack");
+                let trace_writer = std::fs::File::create(trace_file.clone()).unwrap();
+                let trace_writer = GzEncoder::new(trace_writer, flate2::Compression::fast());
+                TraceWriter::new(trace_writer)
+            }
+            Compression::GzipBest => {
+                trace_file = internal_dir.join("trace-turbopack");
+                let trace_writer = std::fs::File::create(trace_file.clone()).unwrap();
+                let trace_writer = GzEncoder::new(trace_writer, flate2::Compression::best());
+                TraceWriter::new(trace_writer)
+            }
+        };
         let subscriber = subscriber.with(RawTraceLayer::new(trace_writer));
 
         exit.on_exit(async move {

turbopack/crates/turbopack-trace-server/Cargo.toml

@@ -14,7 +14,7 @@ bench = false
 [dependencies]
 anyhow = { workspace = true, features = ["backtrace"] }
 either = { workspace = true }
-flate2 = { version = "1.0.28" }
+flate2 = { workspace = true }
 hashbrown = { workspace = true, features = ["raw"] }
 indexmap = { workspace = true, features = ["serde"] }
 itertools = { workspace = true }

turbopack/crates/turbopack-trace-server/src/lib.rs

@@ -1,5 +1,6 @@
 #![feature(iter_intersperse)]
 #![feature(box_patterns)]
+#![feature(bufreader_peek)]
 
 use std::{hash::BuildHasherDefault, path::PathBuf, sync::Arc};
 

turbopack/crates/turbopack-trace-server/src/main.rs

@@ -1,5 +1,6 @@
 #![feature(iter_intersperse)]
 #![feature(box_patterns)]
+#![feature(bufreader_peek)]
 
 use std::{hash::BuildHasherDefault, sync::Arc};
 

turbopack/crates/turbopack-trace-server/src/reader/mod.rs

@@ -75,7 +75,7 @@ impl ObjectSafeTraceFormat for ErasedTraceFormat {
 
 #[derive(Default)]
 enum TraceFile {
-    Raw(File),
+    Raw(BufReader<File>),
     Zstd(zstd::Decoder<'static, BufReader<File>>),
     Gz(GzDecoder<BufReader<File>>),
     #[default]
@@ -112,7 +112,7 @@ impl TraceFile {
 
     fn size(&mut self) -> io::Result<u64> {
         match self {
-            Self::Raw(file) => file.metadata().map(|m| m.len()),
+            Self::Raw(file) => file.get_ref().metadata().map(|m| m.len()),
             Self::Zstd(decoder) => decoder.get_mut().get_ref().metadata().map(|m| m.len()),
             Self::Gz(decoder) => decoder.get_mut().get_ref().metadata().map(|m| m.len()),
             Self::Unloaded => unreachable!(),
@@ -145,13 +145,21 @@ impl TraceReader {
 
     fn trace_file_from_file(&self, file: File) -> io::Result<TraceFile> {
         let path = &self.path.to_string_lossy();
-        Ok(if path.ends_with(".zst") {
-            TraceFile::Zstd(zstd::Decoder::new(file)?)
-        } else if path.ends_with(".gz") {
-            TraceFile::Gz(GzDecoder::new(BufReader::new(file)))
-        } else {
-            TraceFile::Raw(file)
-        })
+        let mut file = BufReader::with_capacity(
+            // zstd max block size (1 << 17) + block header (3) + magic bytes (4)
+            (1 << 17) + 7,
+            file,
+        );
+        let magic_bytes = file.peek(4)?;
+        Ok(
+            if path.ends_with(".zst") || magic_bytes == [0x27, 0xb5, 0x2f, 0xfd] {
+                TraceFile::Zstd(zstd::Decoder::with_buffer(file)?)
+            } else if path.ends_with(".gz") || matches!(magic_bytes, [0x1f, 0x8b, _, _]) {
+                TraceFile::Gz(GzDecoder::new(file))
+            } else {
+                TraceFile::Raw(file)
+            },
+        )
     }
 
     fn try_read(&mut self) -> bool {
@@ -293,7 +301,9 @@ impl TraceReader {
                     }
                 }
                 Err(err) => {
-                    if err.kind() == io::ErrorKind::UnexpectedEof {
+                    if err.kind() == io::ErrorKind::UnexpectedEof
+                        || err.kind() == io::ErrorKind::InvalidInput
+                    {
                         if let Some(value) = self.wait_for_more_data(
                             &mut file,
                             &mut initial_read,