From 7511e361e512dc47ef5c97b36eb1f022749a0ae3 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 9 Oct 2025 12:11:33 +0200 Subject: [PATCH 01/18] update petgraph --- Cargo.lock | 29 ++++++++++++----------------- Cargo.toml | 2 +- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ca84b8c98778f..a883e1e701689 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2234,12 +2234,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - [[package]] name = "fixedbitset" version = "0.5.7" @@ -4983,24 +4977,25 @@ dependencies = [ [[package]] name = "petgraph" -version = "0.6.3" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ - "fixedbitset 0.4.2", - "indexmap 1.9.3", - "serde", - "serde_derive", + "fixedbitset", + "indexmap 2.9.0", ] [[package]] name = "petgraph" -version = "0.7.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ - "fixedbitset 0.5.7", + "fixedbitset", + "hashbrown 0.15.4", "indexmap 2.9.0", + "serde", + "serde_derive", ] [[package]] @@ -9625,7 +9620,7 @@ dependencies = [ "indexmap 2.9.0", "once_cell", "patricia_tree", - "petgraph 0.6.3", + "petgraph 0.8.3", "ref-cast", "regex", "roaring", @@ -9744,7 +9739,7 @@ dependencies = [ "num-traits", "once_cell", "parking_lot", - "petgraph 0.6.3", + "petgraph 0.8.3", "regex", "rustc-hash 2.1.1", "serde", diff --git a/Cargo.toml b/Cargo.toml index 3e13af27fb9b3..0576427a5ef30 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -403,7 +403,7 @@ owo-colors = "4.2.2" parcel_selectors = "0.28.2" parking_lot = "0.12.1" pathdiff = "0.2.1" -petgraph = "0.6.3" +petgraph = "0.8.3" pin-project-lite = "0.2.9" postcard = "1.0.4" proc-macro2 = "1.0.79" From 8f86b1bdafb0091ce12b5d4a617e37e7f9678fcb Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 9 Oct 2025 12:11:51 +0200 Subject: [PATCH 02/18] WIP: enable trace_task_dirty feature --- turbopack/crates/turbo-tasks-backend/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbopack/crates/turbo-tasks-backend/Cargo.toml b/turbopack/crates/turbo-tasks-backend/Cargo.toml index 4513aafb16ca4..024e47ef6e740 100644 --- a/turbopack/crates/turbo-tasks-backend/Cargo.toml +++ b/turbopack/crates/turbo-tasks-backend/Cargo.toml @@ -13,7 +13,7 @@ bench = false workspace = true [features] -default = [] +default = ["trace_task_dirty"] print_cache_item_size = [] no_fast_stale = [] verify_serialization = [] From 0a3f3e90effa504b9a15e393e029556aba33408c Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 9 Oct 2025 12:46:51 +0200 Subject: [PATCH 03/18] remove tracing restriction --- packages/next-swc/package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/next-swc/package.json b/packages/next-swc/package.json index 6ad044b694682..437fb9e919540 100644 --- a/packages/next-swc/package.json +++ b/packages/next-swc/package.json @@ -8,10 +8,10 @@ "scripts": { "clean": "node ../../scripts/rm.mjs native", "build-native": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --features plugin,image-extended --js false native", - "build-native-release": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --release --features plugin,image-extended,tracing/release_max_level_info --js false native", - "build-native-release-with-assertions": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --profile release-with-assertions --features plugin,image-extended,tracing/release_max_level_info --js false native", + "build-native-release": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --release --features plugin,image-extended --js false native", + "build-native-release-with-assertions": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --profile release-with-assertions --features plugin,image-extended --js false native", "build-native-no-plugin": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --features image-webp --js false native", - "build-native-no-plugin-release": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --release --features image-webp,tracing/release_max_level_info --js false native", + "build-native-no-plugin-release": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --release --features image-webp --js false native", "build-native-wasi": "npx --package=@napi-rs/cli@3.0.0-alpha.45 napi build --platform --target wasm32-wasip1-threads -p next-swc-napi --cwd ../../ --output-dir packages/next-swc/native --no-default-features", "build-wasm": "wasm-pack build ../../crates/wasm --scope=next", "cache-build-native": "[ -d native ] && echo $(ls native)", From f10271607fea29a7bccc9c36ce633dc3435bad44 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 9 Oct 2025 15:25:29 +0200 Subject: [PATCH 04/18] Turbopack: take task cache log before starting snapshot --- turbopack/crates/turbo-tasks-backend/src/backend/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index fac0fbdf175ea..ab823703f720f 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -958,12 +958,12 @@ impl TurboTasksBackendInner { .map(|op| op.arc().clone()) .collect::>(); drop(snapshot_request); + self.storage.start_snapshot(); let mut persisted_task_cache_log = self .persisted_task_cache_log .as_ref() .map(|l| l.take(|i| i)) .unwrap_or_default(); - self.storage.start_snapshot(); let mut snapshot_request = self.snapshot_request.lock(); snapshot_request.snapshot_requested = false; self.in_progress_operations From 4f264a74a8283ccc1ae2c8ebb3b0f79ef5180421 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 9 Oct 2025 16:18:20 +0200 Subject: [PATCH 05/18] enable verify_immutable --- turbopack/crates/turbo-tasks-backend/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbopack/crates/turbo-tasks-backend/Cargo.toml b/turbopack/crates/turbo-tasks-backend/Cargo.toml index 024e47ef6e740..64e5b94e01447 100644 --- a/turbopack/crates/turbo-tasks-backend/Cargo.toml +++ b/turbopack/crates/turbo-tasks-backend/Cargo.toml @@ -13,7 +13,7 @@ bench = false workspace = true [features] -default = ["trace_task_dirty"] +default = ["trace_task_dirty", "verify_immutable"] print_cache_item_size = [] no_fast_stale = [] verify_serialization = [] From 9937aea5ca05656c84917a85aa642090073bfea2 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 9 Oct 2025 16:31:36 +0200 Subject: [PATCH 06/18] add more tracing --- .../crates/turbo-tasks-backend/src/kv_backing_storage.rs | 8 ++++++++ .../crates/turbopack-core/src/chunk/available_modules.rs | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs index 76bd40db4cb47..18a09a57b3554 100644 --- a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs @@ -499,12 +499,20 @@ impl BackingStorageSealed tx: &D::ReadTransaction<'_>, task_type: &CachedTaskType, ) -> Result> { + let span = tracing::trace_span!( + "forward_lookup_task_cache", + task_type = debug(task_type), + success = tracing::field::Empty + ) + .entered(); let task_type = POT_CONFIG.serialize(task_type)?; let Some(bytes) = database.get(tx, KeySpace::ForwardTaskCache, &task_type)? else { + span.record("success", false); return Ok(None); }; let bytes = bytes.borrow().try_into()?; let id = TaskId::try_from(u32::from_le_bytes(bytes)).unwrap(); + span.record("success", *id); Ok(Some(id)) } if inner.database.is_empty() { diff --git a/turbopack/crates/turbopack-core/src/chunk/available_modules.rs b/turbopack/crates/turbopack-core/src/chunk/available_modules.rs index fbcc8e972e754..2b5902064cbc0 100644 --- a/turbopack/crates/turbopack-core/src/chunk/available_modules.rs +++ b/turbopack/crates/turbopack-core/src/chunk/available_modules.rs @@ -33,6 +33,12 @@ impl AvailableModules { self: ResolvedVc, modules: ResolvedVc, ) -> Result> { + let _span = tracing::trace_span!( + "AvailabilityInfo::with_modules", + this = debug(self), + modules = debug(&modules) + ) + .entered(); Ok(AvailableModules { parent: Some(self), modules, From 38ae1ee1403332439c1b834b908a9c4894156499 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 10 Oct 2025 09:14:19 +0200 Subject: [PATCH 07/18] verify_serialization --- turbopack/crates/turbo-tasks-backend/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbopack/crates/turbo-tasks-backend/Cargo.toml b/turbopack/crates/turbo-tasks-backend/Cargo.toml index 64e5b94e01447..e38ccf282d3ed 100644 --- a/turbopack/crates/turbo-tasks-backend/Cargo.toml +++ b/turbopack/crates/turbo-tasks-backend/Cargo.toml @@ -13,7 +13,7 @@ bench = false workspace = true [features] -default = ["trace_task_dirty", "verify_immutable"] +default = ["trace_task_dirty", "verify_immutable", "verify_serialization"] print_cache_item_size = [] no_fast_stale = [] verify_serialization = [] From cbdd1eedd2d8ca4971e99992a91da9f0b0f55418 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 10 Oct 2025 09:48:45 +0200 Subject: [PATCH 08/18] Turbopack: Add ChunkedVec test case --- .../src/utils/chunked_vec.rs | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/chunked_vec.rs b/turbopack/crates/turbo-tasks-backend/src/utils/chunked_vec.rs index 245c5344254b8..7666cf6662772 100644 --- a/turbopack/crates/turbo-tasks-backend/src/utils/chunked_vec.rs +++ b/turbopack/crates/turbo-tasks-backend/src/utils/chunked_vec.rs @@ -41,7 +41,7 @@ impl ChunkedVec { } } - pub fn iter(&self) -> impl Iterator { + pub fn iter(&self) -> impl ExactSizeIterator { ExactSizeIter { iter: self.chunks.iter().flat_map(|chunk| chunk.iter()), len: self.len(), @@ -104,3 +104,28 @@ impl ExactSizeIterator for ExactSizeIter { self.len } } + +#[cfg(test)] +mod tests { + use super::ChunkedVec; + + #[test] + fn test_chunked_vec() { + for i in 0..1000 { + let mut vec = ChunkedVec::new(); + for j in 0..i { + vec.push(j); + } + assert_eq!(vec.len(), i); + assert_eq!( + vec.iter().copied().collect::>(), + (0..i).collect::>() + ); + assert_eq!(vec.iter().len(), i); + assert_eq!(vec.is_empty(), i == 0); + let iter = vec.into_iter(); + assert_eq!(iter.len(), i); + assert_eq!(iter.collect::>(), (0..i).collect::>()); + } + } +} From 2750556572b9358a899becdc0b075507b9c880a5 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 10 Oct 2025 09:49:10 +0200 Subject: [PATCH 09/18] improve verify_serialization to check task type round trip --- .../src/kv_backing_storage.rs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs index 18a09a57b3554..ab8e2670af864 100644 --- a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs @@ -667,9 +667,22 @@ fn serialize_task_type( let deserialize: Result = serde_path_to_error::deserialize( &mut pot_de_symbol_list().deserializer_for_slice(&*task_type_bytes)?, ); - if let Err(err) = deserialize { - println!("Task type would not be deserializable {task_id}: {err:?}\n{task_type:#?}"); - panic!("Task type would not be deserializable {task_id}: {err:?}"); + match deserialize { + Ok(value) => { + if value != **task_type { + println!( + "Task type would not round-trip {task_id}:\nOriginal: \ + {task_type:#?}\nRound-tripped: {value:#?}" + ); + panic!("Task type would not round-trip {task_id}"); + } + } + Err(err) => { + println!( + "Task type would not be deserializable {task_id}: {err:?}\n{task_type:#?}" + ); + panic!("Task type would not be deserializable {task_id}: {err:?}"); + } } } Ok(()) From b5403cf03136731dca15707718ab9f3c991b9c9a Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 10 Oct 2025 09:49:35 +0200 Subject: [PATCH 10/18] WIP: print when storing a with_modules task --- turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs index ab8e2670af864..87761a0a53003 100644 --- a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs @@ -359,6 +359,9 @@ impl BackingStorageSealed for (task_type, task_id) in updates { let task_id: u32 = *task_id; serialize_task_type(&task_type, &mut task_type_bytes, task_id)?; + if task_type.get_name().contains("with_modules") { + println!("Stored task type: {task_type:?} => {task_id}"); + } batch .put( From 9d607afecae892309d600078c32a6d8cd37c54a8 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 10 Oct 2025 13:01:03 +0200 Subject: [PATCH 11/18] Revert "WIP: print when storing a with_modules task" This reverts commit b5403cf03136731dca15707718ab9f3c991b9c9a. --- turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs index 87761a0a53003..ab8e2670af864 100644 --- a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs @@ -359,9 +359,6 @@ impl BackingStorageSealed for (task_type, task_id) in updates { let task_id: u32 = *task_id; serialize_task_type(&task_type, &mut task_type_bytes, task_id)?; - if task_type.get_name().contains("with_modules") { - println!("Stored task type: {task_type:?} => {task_id}"); - } batch .put( From 3ccb8c7ccb344a9d8cb65bf6df5bc51f0f70fbcc Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 10 Oct 2025 13:17:46 +0200 Subject: [PATCH 12/18] Revert "verify_serialization" This reverts commit 38ae1ee1403332439c1b834b908a9c4894156499. --- turbopack/crates/turbo-tasks-backend/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbopack/crates/turbo-tasks-backend/Cargo.toml b/turbopack/crates/turbo-tasks-backend/Cargo.toml index e38ccf282d3ed..64e5b94e01447 100644 --- a/turbopack/crates/turbo-tasks-backend/Cargo.toml +++ b/turbopack/crates/turbo-tasks-backend/Cargo.toml @@ -13,7 +13,7 @@ bench = false workspace = true [features] -default = ["trace_task_dirty", "verify_immutable", "verify_serialization"] +default = ["trace_task_dirty", "verify_immutable"] print_cache_item_size = [] no_fast_stale = [] verify_serialization = [] From 9c0774c7bdeb1655e3f9f0513bf80730be73560d Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 10 Oct 2025 09:49:35 +0200 Subject: [PATCH 13/18] WIP: print when storing a with_modules task --- turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs index ab8e2670af864..87761a0a53003 100644 --- a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs @@ -359,6 +359,9 @@ impl BackingStorageSealed for (task_type, task_id) in updates { let task_id: u32 = *task_id; serialize_task_type(&task_type, &mut task_type_bytes, task_id)?; + if task_type.get_name().contains("with_modules") { + println!("Stored task type: {task_type:?} => {task_id}"); + } batch .put( From fc3917b01c0942e863153076a96e14024d991c5c Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 10 Oct 2025 13:53:19 +0200 Subject: [PATCH 14/18] only enable verify_serialization for TaskType --- turbopack/crates/turbo-tasks-backend/Cargo.toml | 6 ++++-- .../crates/turbo-tasks-backend/src/kv_backing_storage.rs | 8 ++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/Cargo.toml b/turbopack/crates/turbo-tasks-backend/Cargo.toml index 64e5b94e01447..fe041f8bc245b 100644 --- a/turbopack/crates/turbo-tasks-backend/Cargo.toml +++ b/turbopack/crates/turbo-tasks-backend/Cargo.toml @@ -13,10 +13,12 @@ bench = false workspace = true [features] -default = ["trace_task_dirty", "verify_immutable"] +default = ["trace_task_dirty", "verify_immutable", "verify_serialization3"] print_cache_item_size = [] no_fast_stale = [] -verify_serialization = [] +verify_serialization3 = [] +verify_serialization2 = [] +verify_serialization1 = [] verify_aggregation_graph = [] verify_immutable = [] trace_aggregation_update = [] diff --git a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs index 87761a0a53003..9e81ca93a0d70 100644 --- a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs @@ -665,7 +665,7 @@ fn serialize_task_type( POT_CONFIG .serialize_into(&**task_type, &mut task_type_bytes) .with_context(|| anyhow!("Unable to serialize task {task_id} cache key {task_type:?}"))?; - #[cfg(feature = "verify_serialization")] + #[cfg(feature = "verify_serialization3")] { let deserialize: Result = serde_path_to_error::deserialize( &mut pot_de_symbol_list().deserializer_for_slice(&*task_type_bytes)?, @@ -749,7 +749,7 @@ where fn serialize(task: TaskId, data: &Vec) -> Result> { Ok(match pot_serialize_small_vec(data) { - #[cfg(not(feature = "verify_serialization"))] + #[cfg(not(feature = "verify_serialization1"))] Ok(value) => value, _ => { let mut error = Ok(()); @@ -760,7 +760,7 @@ fn serialize(task: TaskId, data: &Vec) -> Result) -> Result = serde_path_to_error::deserialize( From 64432f4f3be02298ea45c2a4392d7de832453601 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 10 Oct 2025 14:06:37 +0200 Subject: [PATCH 15/18] add more logging --- .../src/kv_backing_storage.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs index 9e81ca93a0d70..4251bd9efc03c 100644 --- a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs @@ -360,7 +360,10 @@ impl BackingStorageSealed let task_id: u32 = *task_id; serialize_task_type(&task_type, &mut task_type_bytes, task_id)?; if task_type.get_name().contains("with_modules") { - println!("Stored task type: {task_type:?} => {task_id}"); + println!( + "Stored task type: {task_type:?} => {task_id} \ + ({task_type_bytes:?})" + ); } batch @@ -508,13 +511,20 @@ impl BackingStorageSealed success = tracing::field::Empty ) .entered(); - let task_type = POT_CONFIG.serialize(task_type)?; - let Some(bytes) = database.get(tx, KeySpace::ForwardTaskCache, &task_type)? else { + let task_type_bytes = POT_CONFIG.serialize(task_type)?; + let result = database.get(tx, KeySpace::ForwardTaskCache, &task_type_bytes)?; + let Some(bytes) = result else { + if task_type.get_name().contains("with_modules") { + println!("Restored task type: {task_type:?} => None ({task_type_bytes:?})"); + } span.record("success", false); return Ok(None); }; let bytes = bytes.borrow().try_into()?; let id = TaskId::try_from(u32::from_le_bytes(bytes)).unwrap(); + if task_type.get_name().contains("with_modules") { + println!("Restored task type: {task_type:?} => {id} ({task_type_bytes:?})"); + } span.record("success", *id); Ok(Some(id)) } From 0d975ec0a79ccb89b69e3e162a8f8fe01cfba865 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 10 Oct 2025 14:26:14 +0200 Subject: [PATCH 16/18] verify_serialization2 --- turbopack/crates/turbo-tasks-backend/Cargo.toml | 2 +- turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/Cargo.toml b/turbopack/crates/turbo-tasks-backend/Cargo.toml index fe041f8bc245b..2b732e8195277 100644 --- a/turbopack/crates/turbo-tasks-backend/Cargo.toml +++ b/turbopack/crates/turbo-tasks-backend/Cargo.toml @@ -13,7 +13,7 @@ bench = false workspace = true [features] -default = ["trace_task_dirty", "verify_immutable", "verify_serialization3"] +default = ["trace_task_dirty", "verify_immutable", "verify_serialization2"] print_cache_item_size = [] no_fast_stale = [] verify_serialization3 = [] diff --git a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs index 4251bd9efc03c..cd4a0694c19a6 100644 --- a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs @@ -759,7 +759,7 @@ where fn serialize(task: TaskId, data: &Vec) -> Result> { Ok(match pot_serialize_small_vec(data) { - #[cfg(not(feature = "verify_serialization1"))] + #[cfg(not(feature = "verify_serialization2"))] Ok(value) => value, _ => { let mut error = Ok(()); From 194caa78e9122cd3438752d17fc8e14df1a5a43e Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 10 Oct 2025 17:03:38 +0200 Subject: [PATCH 17/18] Turbopack: use the same serialization method for lookup as for storing --- .../src/kv_backing_storage.rs | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs index cd4a0694c19a6..a0854f8c995d6 100644 --- a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs @@ -357,8 +357,12 @@ impl BackingStorageSealed let mut task_type_bytes = Vec::new(); for (task_type, task_id) in updates { + serialize_task_type( + &task_type, + &mut task_type_bytes, + Some(task_id), + )?; let task_id: u32 = *task_id; - serialize_task_type(&task_type, &mut task_type_bytes, task_id)?; if task_type.get_name().contains("with_modules") { println!( "Stored task type: {task_type:?} => {task_id} \ @@ -447,8 +451,8 @@ impl BackingStorageSealed .entered(); let mut task_type_bytes = Vec::new(); for (task_type, task_id) in task_cache_updates.into_iter().flatten() { + serialize_task_type(&task_type, &mut task_type_bytes, Some(task_id))?; let task_id = *task_id; - serialize_task_type(&task_type, &mut task_type_bytes, task_id)?; batch .put( @@ -511,7 +515,8 @@ impl BackingStorageSealed success = tracing::field::Empty ) .entered(); - let task_type_bytes = POT_CONFIG.serialize(task_type)?; + let mut task_type_bytes = Vec::new(); + serialize_task_type(&task_type, &mut task_type_bytes, None)?; let result = database.get(tx, KeySpace::ForwardTaskCache, &task_type_bytes)?; let Some(bytes) = result else { if task_type.get_name().contains("with_modules") { @@ -666,15 +671,22 @@ where Ok(()) } +#[inline(never)] fn serialize_task_type( - task_type: &Arc, + task_type: &CachedTaskType, mut task_type_bytes: &mut Vec, - task_id: u32, + task_id: Option, ) -> Result<()> { task_type_bytes.clear(); POT_CONFIG - .serialize_into(&**task_type, &mut task_type_bytes) - .with_context(|| anyhow!("Unable to serialize task {task_id} cache key {task_type:?}"))?; + .serialize_into(&*task_type, &mut task_type_bytes) + .with_context(|| { + if let Some(task_id) = task_id { + anyhow!("Unable to serialize task {task_id} cache key {task_type:?}") + } else { + anyhow!("Unable to serialize task cache key {task_type:?}") + } + })?; #[cfg(feature = "verify_serialization3")] { let deserialize: Result = serde_path_to_error::deserialize( @@ -682,19 +694,19 @@ fn serialize_task_type( ); match deserialize { Ok(value) => { - if value != **task_type { + if value != *task_type { println!( - "Task type would not round-trip {task_id}:\nOriginal: \ + "Task type would not round-trip {task_id:?}:\nOriginal: \ {task_type:#?}\nRound-tripped: {value:#?}" ); - panic!("Task type would not round-trip {task_id}"); + panic!("Task type would not round-trip {task_id:?}"); } } Err(err) => { println!( - "Task type would not be deserializable {task_id}: {err:?}\n{task_type:#?}" + "Task type would not be deserializable {task_id:?}: {err:?}\n{task_type:#?}" ); - panic!("Task type would not be deserializable {task_id}: {err:?}"); + panic!("Task type would not be deserializable {task_id:?}: {err:?}"); } } } From c79f06f40788018acc1238296f670aad901e2ffa Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 10 Oct 2025 17:06:16 +0200 Subject: [PATCH 18/18] disable verify_serialization2 --- turbopack/crates/turbo-tasks-backend/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbopack/crates/turbo-tasks-backend/Cargo.toml b/turbopack/crates/turbo-tasks-backend/Cargo.toml index 2b732e8195277..2dc901fab00f6 100644 --- a/turbopack/crates/turbo-tasks-backend/Cargo.toml +++ b/turbopack/crates/turbo-tasks-backend/Cargo.toml @@ -13,7 +13,7 @@ bench = false workspace = true [features] -default = ["trace_task_dirty", "verify_immutable", "verify_serialization2"] +default = ["trace_task_dirty", "verify_immutable"] print_cache_item_size = [] no_fast_stale = [] verify_serialization3 = []