[Turbopack] include async loaders in chunking instead of separating it (#75052)

### What?

Chunk async loaders together with normal chunk items to avoid multiple chunkings and small chunks

Author: sokra

turbopack/crates/turbopack-core/src/chunk/chunk_group.rs

@@ -163,7 +163,6 @@ pub async fn make_chunk_group(
         })
         .try_join()
         .await?;
-    let has_async_loaders = !async_loaders.is_empty();
     let async_loader_chunk_items =
         async_loaders
             .iter()
@@ -180,14 +179,8 @@ pub async fn make_chunk_group(
         .map(|&loader| loader.references())
         .try_join()
         .await?;
-    let async_loader_external_module_references = Vc::cell(
-        async_loader_references
-            .iter()
-            .flat_map(|references| references.iter().copied())
-            .collect(),
-    );
 
-    let traced_output_assets = traced_modules
+    let mut referenced_output_assets = traced_modules
         .into_iter()
         .map(|module| async move {
             Ok(ResolvedVc::upcast(
@@ -203,7 +196,7 @@ pub async fn make_chunk_group(
         .try_join()
         .await?;
 
-    let chunk_items = all_modules
+    let mut chunk_items = all_modules
         .iter()
         .map(|(module, async_info)| {
             Either::Left(async move {
@@ -234,34 +227,21 @@ pub async fn make_chunk_group(
         .try_join()
         .await?;
 
+    chunk_items.extend(async_loader_chunk_items);
+    referenced_output_assets.reserve(async_loader_references.iter().map(|r| r.len()).sum());
+    referenced_output_assets.extend(async_loader_references.into_iter().flatten());
+
     // Pass chunk items to chunking algorithm
-    let mut chunks = make_chunks(
+    let chunks = make_chunks(
         module_graph,
         *chunking_context,
         Vc::cell(chunk_items),
         "".into(),
-        Vc::cell(traced_output_assets),
+        Vc::cell(referenced_output_assets),
     )
     .await?
     .clone_value();
 
-    if has_async_loaders {
-        // Pass async chunk loaders to chunking algorithm
-        // We want them to be separate since they are specific to this chunk group due
-        // to available chunk items differing
-        let async_loader_chunks = make_chunks(
-            module_graph,
-            *chunking_context,
-            Vc::cell(async_loader_chunk_items.into_iter().collect()),
-            "async-loader-".into(),
-            async_loader_external_module_references,
-        )
-        .await?;
-
-        // concatenate chunks
-        chunks.extend(async_loader_chunks.iter().copied());
-    }
-
     Ok(MakeChunkGroupResult {
         chunks,
         availability_info,

turbopack/crates/turbopack-core/src/chunk/chunking/dev.rs

@@ -39,16 +39,29 @@ pub async fn app_vendors_split(
     mut name: String,
     split_context: &mut SplitContext<'_>,
 ) -> Result<()> {
+    let mut chunk_group_specific_chunk_items = Vec::new();
     let mut app_chunk_items = Vec::new();
     let mut vendors_chunk_items = Vec::new();
     for item in chunk_items {
-        let ChunkItemWithInfo { asset_ident, .. } = &item;
-        if is_app_code(asset_ident) {
+        let ChunkItemWithInfo {
+            asset_ident,
+            module,
+            ..
+        } = &item;
+        if module.is_none() {
+            // This happens for async module loaders.
+            // We want them to be in a separate chunk.
+            chunk_group_specific_chunk_items.push(item);
+        } else if is_app_code(asset_ident) {
             app_chunk_items.push(item);
         } else {
             vendors_chunk_items.push(item);
         }
     }
+    if !chunk_group_specific_chunk_items.is_empty() {
+        let mut name = format!("{}-specific", name);
+        make_chunk(chunk_group_specific_chunk_items, &mut name, split_context).await?;
+    }
     let mut remaining = Vec::new();
     let mut key = format!("{}-app", name);
     if !handle_split_group(

turbopack/crates/turbopack-tests/tests/snapshot/basic-tree-shake/dynamic-import/output/4c35f_tests_snapshot_basic-tree-shake_dynamic-import_input_index_aba1a0.js

@@ -0,0 +1,6 @@
+(globalThis.TURBOPACK = globalThis.TURBOPACK || []).push([
+    "output/4c35f_tests_snapshot_basic-tree-shake_dynamic-import_input_index_aba1a0.js",
+    {},
+    {"otherChunks":["output/b1abf_turbopack-tests_tests_snapshot_basic-tree-shake_dynamic-import_input_lib_44a9b7.js","output/4c35f_tests_snapshot_basic-tree-shake_dynamic-import_input_index_92a5f4.js"],"runtimeModuleIds":["[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic-tree-shake/dynamic-import/input/index.js [test] (ecmascript)"]}
+]);
+// Dummy runtime

turbopack/crates/turbopack-tests/tests/snapshot/basic-tree-shake/dynamic-import/output/4c35f_tests_snapshot_basic-tree-shake_dynamic-import_input_index_d5474b.js.map renamed to turbopack/crates/turbopack-tests/tests/snapshot/basic-tree-shake/dynamic-import/output/4c35f_tests_snapshot_basic-tree-shake_dynamic-import_input_index_aba1a0.js.map

@@ -0,0 +1,6 @@
+(globalThis.TURBOPACK = globalThis.TURBOPACK || []).push([
+    "output/4e721_crates_turbopack-tests_tests_snapshot_basic_async_chunk_input_index_c2e2cc.js",
+    {},
+    {"otherChunks":["output/4e721_crates_turbopack-tests_tests_snapshot_basic_async_chunk_input_import_46e42b.js","output/turbopack_crates_turbopack-tests_tests_snapshot_basic_async_chunk_input_463663._.js"],"runtimeModuleIds":["[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk/input/index.js [test] (ecmascript)"]}
+]);
+// Dummy runtime