diff --git a/Cargo.lock b/Cargo.lock index ca84b8c98778f7..a883e1e7016898 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2234,12 +2234,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - [[package]] name = "fixedbitset" version = "0.5.7" @@ -4983,24 +4977,25 @@ dependencies = [ [[package]] name = "petgraph" -version = "0.6.3" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ - "fixedbitset 0.4.2", - "indexmap 1.9.3", - "serde", - "serde_derive", + "fixedbitset", + "indexmap 2.9.0", ] [[package]] name = "petgraph" -version = "0.7.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ - "fixedbitset 0.5.7", + "fixedbitset", + "hashbrown 0.15.4", "indexmap 2.9.0", + "serde", + "serde_derive", ] [[package]] @@ -9625,7 +9620,7 @@ dependencies = [ "indexmap 2.9.0", "once_cell", "patricia_tree", - "petgraph 0.6.3", + "petgraph 0.8.3", "ref-cast", "regex", "roaring", @@ -9744,7 +9739,7 @@ dependencies = [ "num-traits", "once_cell", "parking_lot", - "petgraph 0.6.3", + "petgraph 0.8.3", "regex", "rustc-hash 2.1.1", "serde", diff --git a/Cargo.toml b/Cargo.toml index 3e13af27fb9b36..0576427a5ef30f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -403,7 +403,7 @@ owo-colors = "4.2.2" parcel_selectors = "0.28.2" parking_lot = "0.12.1" pathdiff = "0.2.1" -petgraph = "0.6.3" +petgraph = "0.8.3" pin-project-lite = "0.2.9" postcard = "1.0.4" proc-macro2 = "1.0.79" diff --git a/packages/next-swc/package.json b/packages/next-swc/package.json index 6ad044b6946824..437fb9e919540f 100644 --- a/packages/next-swc/package.json +++ b/packages/next-swc/package.json @@ -8,10 +8,10 @@ "scripts": { "clean": "node ../../scripts/rm.mjs native", "build-native": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --features plugin,image-extended --js false native", - "build-native-release": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --release --features plugin,image-extended,tracing/release_max_level_info --js false native", - "build-native-release-with-assertions": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --profile release-with-assertions --features plugin,image-extended,tracing/release_max_level_info --js false native", + "build-native-release": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --release --features plugin,image-extended --js false native", + "build-native-release-with-assertions": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --profile release-with-assertions --features plugin,image-extended --js false native", "build-native-no-plugin": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --features image-webp --js false native", - "build-native-no-plugin-release": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --release --features image-webp,tracing/release_max_level_info --js false native", + "build-native-no-plugin-release": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --release --features image-webp --js false native", "build-native-wasi": "npx --package=@napi-rs/cli@3.0.0-alpha.45 napi build --platform --target wasm32-wasip1-threads -p next-swc-napi --cwd ../../ --output-dir packages/next-swc/native --no-default-features", "build-wasm": "wasm-pack build ../../crates/wasm --scope=next", "cache-build-native": "[ -d native ] && echo $(ls native)", diff --git a/turbopack/crates/turbo-tasks-backend/Cargo.toml b/turbopack/crates/turbo-tasks-backend/Cargo.toml index 4513aafb16ca49..2dc901fab00f69 100644 --- a/turbopack/crates/turbo-tasks-backend/Cargo.toml +++ b/turbopack/crates/turbo-tasks-backend/Cargo.toml @@ -13,10 +13,12 @@ bench = false workspace = true [features] -default = [] +default = ["trace_task_dirty", "verify_immutable"] print_cache_item_size = [] no_fast_stale = [] -verify_serialization = [] +verify_serialization3 = [] +verify_serialization2 = [] +verify_serialization1 = [] verify_aggregation_graph = [] verify_immutable = [] trace_aggregation_update = [] diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index fac0fbdf175ea2..ab823703f720f0 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -958,12 +958,12 @@ impl TurboTasksBackendInner { .map(|op| op.arc().clone()) .collect::>(); drop(snapshot_request); + self.storage.start_snapshot(); let mut persisted_task_cache_log = self .persisted_task_cache_log .as_ref() .map(|l| l.take(|i| i)) .unwrap_or_default(); - self.storage.start_snapshot(); let mut snapshot_request = self.snapshot_request.lock(); snapshot_request.snapshot_requested = false; self.in_progress_operations diff --git a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs index 76bd40db4cb47b..a0854f8c995d63 100644 --- a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs @@ -357,8 +357,18 @@ impl BackingStorageSealed let mut task_type_bytes = Vec::new(); for (task_type, task_id) in updates { + serialize_task_type( + &task_type, + &mut task_type_bytes, + Some(task_id), + )?; let task_id: u32 = *task_id; - serialize_task_type(&task_type, &mut task_type_bytes, task_id)?; + if task_type.get_name().contains("with_modules") { + println!( + "Stored task type: {task_type:?} => {task_id} \ + ({task_type_bytes:?})" + ); + } batch .put( @@ -441,8 +451,8 @@ impl BackingStorageSealed .entered(); let mut task_type_bytes = Vec::new(); for (task_type, task_id) in task_cache_updates.into_iter().flatten() { + serialize_task_type(&task_type, &mut task_type_bytes, Some(task_id))?; let task_id = *task_id; - serialize_task_type(&task_type, &mut task_type_bytes, task_id)?; batch .put( @@ -499,12 +509,28 @@ impl BackingStorageSealed tx: &D::ReadTransaction<'_>, task_type: &CachedTaskType, ) -> Result> { - let task_type = POT_CONFIG.serialize(task_type)?; - let Some(bytes) = database.get(tx, KeySpace::ForwardTaskCache, &task_type)? else { + let span = tracing::trace_span!( + "forward_lookup_task_cache", + task_type = debug(task_type), + success = tracing::field::Empty + ) + .entered(); + let mut task_type_bytes = Vec::new(); + serialize_task_type(&task_type, &mut task_type_bytes, None)?; + let result = database.get(tx, KeySpace::ForwardTaskCache, &task_type_bytes)?; + let Some(bytes) = result else { + if task_type.get_name().contains("with_modules") { + println!("Restored task type: {task_type:?} => None ({task_type_bytes:?})"); + } + span.record("success", false); return Ok(None); }; let bytes = bytes.borrow().try_into()?; let id = TaskId::try_from(u32::from_le_bytes(bytes)).unwrap(); + if task_type.get_name().contains("with_modules") { + println!("Restored task type: {task_type:?} => {id} ({task_type_bytes:?})"); + } + span.record("success", *id); Ok(Some(id)) } if inner.database.is_empty() { @@ -645,23 +671,43 @@ where Ok(()) } +#[inline(never)] fn serialize_task_type( - task_type: &Arc, + task_type: &CachedTaskType, mut task_type_bytes: &mut Vec, - task_id: u32, + task_id: Option, ) -> Result<()> { task_type_bytes.clear(); POT_CONFIG - .serialize_into(&**task_type, &mut task_type_bytes) - .with_context(|| anyhow!("Unable to serialize task {task_id} cache key {task_type:?}"))?; - #[cfg(feature = "verify_serialization")] + .serialize_into(&*task_type, &mut task_type_bytes) + .with_context(|| { + if let Some(task_id) = task_id { + anyhow!("Unable to serialize task {task_id} cache key {task_type:?}") + } else { + anyhow!("Unable to serialize task cache key {task_type:?}") + } + })?; + #[cfg(feature = "verify_serialization3")] { let deserialize: Result = serde_path_to_error::deserialize( &mut pot_de_symbol_list().deserializer_for_slice(&*task_type_bytes)?, ); - if let Err(err) = deserialize { - println!("Task type would not be deserializable {task_id}: {err:?}\n{task_type:#?}"); - panic!("Task type would not be deserializable {task_id}: {err:?}"); + match deserialize { + Ok(value) => { + if value != *task_type { + println!( + "Task type would not round-trip {task_id:?}:\nOriginal: \ + {task_type:#?}\nRound-tripped: {value:#?}" + ); + panic!("Task type would not round-trip {task_id:?}"); + } + } + Err(err) => { + println!( + "Task type would not be deserializable {task_id:?}: {err:?}\n{task_type:#?}" + ); + panic!("Task type would not be deserializable {task_id:?}: {err:?}"); + } } } Ok(()) @@ -725,7 +771,7 @@ where fn serialize(task: TaskId, data: &Vec) -> Result> { Ok(match pot_serialize_small_vec(data) { - #[cfg(not(feature = "verify_serialization"))] + #[cfg(not(feature = "verify_serialization2"))] Ok(value) => value, _ => { let mut error = Ok(()); @@ -736,7 +782,7 @@ fn serialize(task: TaskId, data: &Vec) -> Result) -> Result = serde_path_to_error::deserialize( diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/chunked_vec.rs b/turbopack/crates/turbo-tasks-backend/src/utils/chunked_vec.rs index 245c5344254b84..7666cf6662772d 100644 --- a/turbopack/crates/turbo-tasks-backend/src/utils/chunked_vec.rs +++ b/turbopack/crates/turbo-tasks-backend/src/utils/chunked_vec.rs @@ -41,7 +41,7 @@ impl ChunkedVec { } } - pub fn iter(&self) -> impl Iterator { + pub fn iter(&self) -> impl ExactSizeIterator { ExactSizeIter { iter: self.chunks.iter().flat_map(|chunk| chunk.iter()), len: self.len(), @@ -104,3 +104,28 @@ impl ExactSizeIterator for ExactSizeIter { self.len } } + +#[cfg(test)] +mod tests { + use super::ChunkedVec; + + #[test] + fn test_chunked_vec() { + for i in 0..1000 { + let mut vec = ChunkedVec::new(); + for j in 0..i { + vec.push(j); + } + assert_eq!(vec.len(), i); + assert_eq!( + vec.iter().copied().collect::>(), + (0..i).collect::>() + ); + assert_eq!(vec.iter().len(), i); + assert_eq!(vec.is_empty(), i == 0); + let iter = vec.into_iter(); + assert_eq!(iter.len(), i); + assert_eq!(iter.collect::>(), (0..i).collect::>()); + } + } +} diff --git a/turbopack/crates/turbopack-core/src/chunk/available_modules.rs b/turbopack/crates/turbopack-core/src/chunk/available_modules.rs index fbcc8e972e754f..2b5902064cbc00 100644 --- a/turbopack/crates/turbopack-core/src/chunk/available_modules.rs +++ b/turbopack/crates/turbopack-core/src/chunk/available_modules.rs @@ -33,6 +33,12 @@ impl AvailableModules { self: ResolvedVc, modules: ResolvedVc, ) -> Result> { + let _span = tracing::trace_span!( + "AvailabilityInfo::with_modules", + this = debug(self), + modules = debug(&modules) + ) + .entered(); Ok(AvailableModules { parent: Some(self), modules,