diff --git a/Cargo.lock b/Cargo.lock index ea64ac1c9536e6..ca84b8c98778f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4402,6 +4402,7 @@ dependencies = [ "console-subscriber", "dhat", "either", + "flate2", "futures-util", "getrandom 0.2.15", "iana-time-zone", diff --git a/Cargo.toml b/Cargo.toml index 6c145aa2db38d6..3e13af27fb9b36 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -367,6 +367,7 @@ dhat = { version = "0.3.2" } dunce = "1.0.3" either = "1.9.0" erased-serde = "0.4.5" +flate2 = "1.0.28" futures = "0.3.31" futures-util = "0.3.31" futures-retry = "0.6.0" diff --git a/crates/napi/Cargo.toml b/crates/napi/Cargo.toml index f78a8112914485..a0a3ddec1f0397 100644 --- a/crates/napi/Cargo.toml +++ b/crates/napi/Cargo.toml @@ -55,10 +55,11 @@ ignored = [ ] [dependencies] -anyhow = "1.0.66" +anyhow = { workspace = true } console-subscriber = { workspace = true, optional = true } dhat = { workspace = true, optional = true } either = { workspace = true } +flate2 = { workspace = true } futures-util = { workspace = true } owo-colors = { workspace = true } napi = { workspace = true } diff --git a/crates/napi/src/lib.rs b/crates/napi/src/lib.rs index dbb00d0591032e..24dacb72b44da2 100644 --- a/crates/napi/src/lib.rs +++ b/crates/napi/src/lib.rs @@ -30,6 +30,7 @@ DEALINGS IN THE SOFTWARE. //#![deny(clippy::all)] #![feature(arbitrary_self_types)] #![feature(arbitrary_self_types_pointers)] +#![feature(iter_intersperse)] #[macro_use] extern crate napi_derive; diff --git a/crates/napi/src/next_api/project.rs b/crates/napi/src/next_api/project.rs index 527b1573b467c4..c32e785aaa296e 100644 --- a/crates/napi/src/next_api/project.rs +++ b/crates/napi/src/next_api/project.rs @@ -1,6 +1,7 @@ use std::{borrow::Cow, io::Write, path::PathBuf, sync::Arc, thread, time::Duration}; use anyhow::{Context, Result, anyhow, bail}; +use flate2::write::GzEncoder; use futures_util::TryFutureExt; use napi::{ Env, JsFunction, JsObject, Status, @@ -363,21 +364,33 @@ pub fn project_new( } if let Some(mut trace) = trace { - // Trace presets - match trace.as_str() { - "overview" | "1" => { - trace = TRACING_NEXT_OVERVIEW_TARGETS.join(","); - } - "next" => { - trace = TRACING_NEXT_TARGETS.join(","); - } - "turbopack" => { - trace = TRACING_NEXT_TURBOPACK_TARGETS.join(","); - } - "turbo-tasks" => { - trace = TRACING_NEXT_TURBO_TASKS_TARGETS.join(","); - } - _ => {} + trace = trace + .split(",") + .map(|item| { + // Trace presets + match item { + "overview" | "1" => Cow::Owned(TRACING_NEXT_OVERVIEW_TARGETS.join(",")), + "next" => Cow::Owned(TRACING_NEXT_TARGETS.join(",")), + "turbopack" => Cow::Owned(TRACING_NEXT_TURBOPACK_TARGETS.join(",")), + "turbo-tasks" => Cow::Owned(TRACING_NEXT_TURBO_TASKS_TARGETS.join(",")), + _ => Cow::Borrowed(item), + } + }) + .intersperse_with(|| Cow::Borrowed(",")) + .collect::(); + + enum Compression { + None, + GzipFast, + GzipBest, + } + let mut compress = Compression::None; + if trace.ends_with(",gz") { + trace.drain(trace.len() - 3..); + compress = Compression::GzipFast; + } else if trace.ends_with(",gz-best") { + trace.drain(trace.len() - 8..); + compress = Compression::GzipBest; } let subscriber = Registry::default(); @@ -396,9 +409,26 @@ pub fn project_new( std::fs::create_dir_all(&internal_dir) .context("Unable to create .next directory") .unwrap(); - let trace_file = internal_dir.join("trace-turbopack"); - let trace_writer = std::fs::File::create(trace_file.clone()).unwrap(); - let (trace_writer, trace_writer_guard) = TraceWriter::new(trace_writer); + let trace_file; + let (trace_writer, trace_writer_guard) = match compress { + Compression::None => { + trace_file = internal_dir.join("trace-turbopack"); + let trace_writer = std::fs::File::create(trace_file.clone()).unwrap(); + TraceWriter::new(trace_writer) + } + Compression::GzipFast => { + trace_file = internal_dir.join("trace-turbopack"); + let trace_writer = std::fs::File::create(trace_file.clone()).unwrap(); + let trace_writer = GzEncoder::new(trace_writer, flate2::Compression::fast()); + TraceWriter::new(trace_writer) + } + Compression::GzipBest => { + trace_file = internal_dir.join("trace-turbopack"); + let trace_writer = std::fs::File::create(trace_file.clone()).unwrap(); + let trace_writer = GzEncoder::new(trace_writer, flate2::Compression::best()); + TraceWriter::new(trace_writer) + } + }; let subscriber = subscriber.with(RawTraceLayer::new(trace_writer)); exit.on_exit(async move { diff --git a/turbopack/crates/turbopack-trace-server/Cargo.toml b/turbopack/crates/turbopack-trace-server/Cargo.toml index 9de0865dda8f0e..c98c3d46d281be 100644 --- a/turbopack/crates/turbopack-trace-server/Cargo.toml +++ b/turbopack/crates/turbopack-trace-server/Cargo.toml @@ -14,7 +14,7 @@ bench = false [dependencies] anyhow = { workspace = true, features = ["backtrace"] } either = { workspace = true } -flate2 = { version = "1.0.28" } +flate2 = { workspace = true } hashbrown = { workspace = true, features = ["raw"] } indexmap = { workspace = true, features = ["serde"] } itertools = { workspace = true } diff --git a/turbopack/crates/turbopack-trace-server/src/lib.rs b/turbopack/crates/turbopack-trace-server/src/lib.rs index 22c35874698ab7..e36464a2794e67 100644 --- a/turbopack/crates/turbopack-trace-server/src/lib.rs +++ b/turbopack/crates/turbopack-trace-server/src/lib.rs @@ -1,5 +1,6 @@ #![feature(iter_intersperse)] #![feature(box_patterns)] +#![feature(bufreader_peek)] use std::{hash::BuildHasherDefault, path::PathBuf, sync::Arc}; diff --git a/turbopack/crates/turbopack-trace-server/src/main.rs b/turbopack/crates/turbopack-trace-server/src/main.rs index 7800b96a8d6ad9..c9145ad5ce94ae 100644 --- a/turbopack/crates/turbopack-trace-server/src/main.rs +++ b/turbopack/crates/turbopack-trace-server/src/main.rs @@ -1,5 +1,6 @@ #![feature(iter_intersperse)] #![feature(box_patterns)] +#![feature(bufreader_peek)] use std::{hash::BuildHasherDefault, sync::Arc}; diff --git a/turbopack/crates/turbopack-trace-server/src/reader/mod.rs b/turbopack/crates/turbopack-trace-server/src/reader/mod.rs index 21a374b96a76ed..eafa552d05cd88 100644 --- a/turbopack/crates/turbopack-trace-server/src/reader/mod.rs +++ b/turbopack/crates/turbopack-trace-server/src/reader/mod.rs @@ -75,7 +75,7 @@ impl ObjectSafeTraceFormat for ErasedTraceFormat { #[derive(Default)] enum TraceFile { - Raw(File), + Raw(BufReader), Zstd(zstd::Decoder<'static, BufReader>), Gz(GzDecoder>), #[default] @@ -112,7 +112,7 @@ impl TraceFile { fn size(&mut self) -> io::Result { match self { - Self::Raw(file) => file.metadata().map(|m| m.len()), + Self::Raw(file) => file.get_ref().metadata().map(|m| m.len()), Self::Zstd(decoder) => decoder.get_mut().get_ref().metadata().map(|m| m.len()), Self::Gz(decoder) => decoder.get_mut().get_ref().metadata().map(|m| m.len()), Self::Unloaded => unreachable!(), @@ -145,13 +145,21 @@ impl TraceReader { fn trace_file_from_file(&self, file: File) -> io::Result { let path = &self.path.to_string_lossy(); - Ok(if path.ends_with(".zst") { - TraceFile::Zstd(zstd::Decoder::new(file)?) - } else if path.ends_with(".gz") { - TraceFile::Gz(GzDecoder::new(BufReader::new(file))) - } else { - TraceFile::Raw(file) - }) + let mut file = BufReader::with_capacity( + // zstd max block size (1 << 17) + block header (3) + magic bytes (4) + (1 << 17) + 7, + file, + ); + let magic_bytes = file.peek(4)?; + Ok( + if path.ends_with(".zst") || magic_bytes == [0x27, 0xb5, 0x2f, 0xfd] { + TraceFile::Zstd(zstd::Decoder::with_buffer(file)?) + } else if path.ends_with(".gz") || matches!(magic_bytes, [0x1f, 0x8b, _, _]) { + TraceFile::Gz(GzDecoder::new(file)) + } else { + TraceFile::Raw(file) + }, + ) } fn try_read(&mut self) -> bool { @@ -293,7 +301,9 @@ impl TraceReader { } } Err(err) => { - if err.kind() == io::ErrorKind::UnexpectedEof { + if err.kind() == io::ErrorKind::UnexpectedEof + || err.kind() == io::ErrorKind::InvalidInput + { if let Some(value) = self.wait_for_more_data( &mut file, &mut initial_read,