Turbopack: migrate global module ids to single-graph (#73752)

Author: mischnic

Cargo.lock

@@ -28,6 +28,7 @@ turbo-rcstr = { workspace = true }
 turbo-tasks = { workspace = true }
 turbo-tasks-env = { workspace = true }
 turbo-tasks-fs = { workspace = true }
+turbo-tasks-hash = { workspace = true }
 turbo-tasks-memory = { workspace = true }
 turbopack = { workspace = true }
 turbopack-browser = { workspace = true }

crates/next-api/Cargo.toml

@@ -9,7 +9,6 @@ mod dynamic_imports;
 mod empty;
 pub mod entrypoints;
 mod font;
-pub mod global_module_id_strategy;
 mod instrumentation;
 mod loadable_manifest;
 mod middleware;

crates/next-api/src/global_module_id_strategy.rs

@@ -86,6 +86,11 @@ pub struct PagesProject {
     project: ResolvedVc<Project>,
 }
 
+#[turbo_tasks::function]
+fn client_layer() -> Vc<RcStr> {
+    Vc::cell("client".into())
+}
+
 #[turbo_tasks::value_impl]
 impl PagesProject {
     #[turbo_tasks::function]
@@ -312,7 +317,7 @@ impl PagesProject {
             self.project().client_compile_time_info(),
             self.client_module_options_context(),
             self.client_resolve_options_context(),
-            Vc::cell("client".into()),
+            client_layer(),
         )
     }
 
@@ -351,7 +356,7 @@ impl PagesProject {
             self.project().client_compile_time_info(),
             self.client_module_options_context(),
             self.client_resolve_options_context(),
-            Vc::cell("client".into()),
+            client_layer(),
         ))
     }
 

crates/next-api/src/lib.rs

@@ -34,7 +34,8 @@ use turbo_tasks::{
 use turbo_tasks_env::{EnvMap, ProcessEnv};
 use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath, VirtualFileSystem};
 use turbopack::{
-    evaluate_context::node_build_environment, transition::TransitionOptions, ModuleAssetContext,
+    evaluate_context::node_build_environment, global_module_ids::get_global_module_id_strategy,
+    transition::TransitionOptions, ModuleAssetContext,
 };
 use turbopack_core::{
     changed::content_changed,
@@ -68,7 +69,6 @@ use crate::{
     build,
     empty::EmptyEndpoint,
     entrypoints::Entrypoints,
-    global_module_id_strategy::GlobalModuleIdStrategyBuilder,
     instrumentation::InstrumentationEndpoint,
     middleware::MiddlewareEndpoint,
     pages::PagesProject,
@@ -1576,16 +1576,25 @@ impl Project {
     /// Gets the module id strategy for the project.
     #[turbo_tasks::function]
     pub async fn module_id_strategy(self: Vc<Self>) -> Result<Vc<Box<dyn ModuleIdStrategy>>> {
-        let module_id_strategy = self.next_config().module_id_strategy_config();
-        match *module_id_strategy.await? {
-            Some(ModuleIdStrategyConfig::Named) => Ok(Vc::upcast(DevModuleIdStrategy::new())),
-            Some(ModuleIdStrategyConfig::Deterministic) => {
-                Ok(Vc::upcast(GlobalModuleIdStrategyBuilder::build(self)))
+        let module_id_strategy = if let Some(module_id_strategy) =
+            &*self.next_config().module_id_strategy_config().await?
+        {
+            *module_id_strategy
+        } else {
+            match *self.next_mode().await? {
+                NextMode::Development => ModuleIdStrategyConfig::Named,
+                NextMode::Build => ModuleIdStrategyConfig::Deterministic,
+            }
+        };
+
+        match module_id_strategy {
+            ModuleIdStrategyConfig::Named => Ok(Vc::upcast(DevModuleIdStrategy::new())),
+            ModuleIdStrategyConfig::Deterministic => {
+                let module_graphs = self.whole_app_module_graphs().await?;
+                Ok(Vc::upcast(get_global_module_id_strategy(
+                    *module_graphs.full,
+                )))
             }
-            None => match *self.next_mode().await? {
-                NextMode::Development => Ok(Vc::upcast(DevModuleIdStrategy::new())),
-                NextMode::Build => Ok(Vc::upcast(DevModuleIdStrategy::new())),
-            },
         }
     }
 }

crates/next-api/src/pages.rs

@@ -567,7 +567,7 @@ pub enum LoaderItem {
 }
 
 #[turbo_tasks::value(operation)]
-#[derive(Clone, Debug)]
+#[derive(Copy, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
 pub enum ModuleIdStrategy {
     Named,
@@ -1483,11 +1483,11 @@ impl NextConfig {
             .experimental
             .turbo
             .as_ref()
-            .and_then(|t| t.module_id_strategy.as_ref())
+            .and_then(|t| t.module_id_strategy)
         else {
             return Vc::cell(None);
         };
-        Vc::cell(Some(module_id_strategy.clone()))
+        Vc::cell(Some(module_id_strategy))
     }
 
     #[turbo_tasks::function]

crates/next-api/src/project.rs

@@ -9,7 +9,7 @@ use turbopack_core::{
     asset::{Asset, AssetContent},
     chunk::{ChunkItem, ChunkItemExt, ChunkType, ChunkableModule, ChunkingContext},
     ident::AssetIdent,
-    module::{Module, Modules},
+    module::Module,
     module_graph::ModuleGraph,
     reference::ModuleReferences,
 };
@@ -18,17 +18,11 @@ use turbopack_ecmascript::{
         EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkPlaceable,
         EcmascriptChunkType, EcmascriptExports,
     },
-    references::{
-        esm::{EsmExport, EsmExports},
-        external_module::IncludeIdentModule,
-    },
+    references::esm::{EsmExport, EsmExports},
     utils::StringifyJs,
 };
 
 use super::server_component_reference::NextServerComponentModuleReference;
-use crate::next_app::app_client_references_chunks::{
-    client_modules_modifier, ssr_modules_modifier,
-};
 
 #[turbo_tasks::function]
 fn modifier() -> Vc<RcStr> {
@@ -68,22 +62,6 @@ impl Module for NextServerComponentModule {
                 .await?,
         )]))
     }
-
-    #[turbo_tasks::function]
-    async fn additional_layers_modules(self: Vc<Self>) -> Result<Vc<Modules>> {
-        let base_ident = self.ident();
-        let ssr_entry_module = ResolvedVc::upcast(
-            IncludeIdentModule::new(base_ident.with_modifier(ssr_modules_modifier()))
-                .to_resolved()
-                .await?,
-        );
-        let client_entry_module = ResolvedVc::upcast(
-            IncludeIdentModule::new(base_ident.with_modifier(client_modules_modifier()))
-                .to_resolved()
-                .await?,
-        );
-        Ok(Vc::cell(vec![ssr_entry_module, client_entry_module]))
-    }
 }
 
 #[turbo_tasks::value_impl]

crates/next-core/src/next_config.rs

@@ -580,16 +580,19 @@ describe('Production Usage', () => {
 
       const resources: Set<string> = new Set()
 
-      const manifestKey = Object.keys(reactLoadableManifest).find((item) => {
-        return item
-          .replace(/\\/g, '/')
-          .endsWith(
-            process.env.TURBOPACK
-              ? 'components/dynamic-css/with-css.js [client] (ecmascript, next/dynamic entry)'
-              : 'dynamic/css.js -> ../../components/dynamic-css/with-css'
-          )
-      })
-      expect(manifestKey).toBeString()
+      let manifestKey: string
+      if (process.env.TURBOPACK) {
+        // the key is an arbitrary and changing number for Turbopack prod, but each page has its own manifest
+        expect(Object.keys(reactLoadableManifest).length).toBe(1)
+        manifestKey = Object.keys(reactLoadableManifest)[0]
+        expect(manifestKey).toBeString()
+      } else {
+        manifestKey = Object.keys(reactLoadableManifest).find((item) =>
+          item
+            .replace(/\\/g, '/')
+            .endsWith('dynamic/css.js -> ../../components/dynamic-css/with-css')
+        )
+      }
 
       // test dynamic chunk
       reactLoadableManifest[manifestKey].files.forEach((f) => {

crates/next-core/src/next_server_component/server_component_module.rs

@@ -1,10 +1,13 @@
-use anyhow::Result;
-use turbo_rcstr::RcStr;
-use turbo_tasks::{FxIndexMap, ResolvedVc, ValueToString, Vc};
+use anyhow::{bail, Result};
+use rustc_hash::FxHashMap;
+use turbo_tasks::{ResolvedVc, ValueToString, Vc};
 use turbo_tasks_hash::hash_xxh3_hash64;
 
 use super::ModuleId;
-use crate::ident::AssetIdent;
+use crate::{
+    ident::AssetIdent,
+    issue::{module::ModuleIssue, IssueExt, StyledString},
+};
 
 #[turbo_tasks::value_trait]
 pub trait ModuleIdStrategy {
@@ -32,25 +35,43 @@ impl ModuleIdStrategy for DevModuleIdStrategy {
     }
 }
 
-#[turbo_tasks::value]
+#[turbo_tasks::value(shared)]
 pub struct GlobalModuleIdStrategy {
-    module_id_map: FxIndexMap<RcStr, ModuleId>,
-}
-
-impl GlobalModuleIdStrategy {
-    pub async fn new(module_id_map: FxIndexMap<RcStr, ModuleId>) -> Result<Vc<Self>> {
-        Ok(GlobalModuleIdStrategy { module_id_map }.cell())
-    }
+    pub module_id_map: FxHashMap<ResolvedVc<AssetIdent>, u64>,
 }
 
 #[turbo_tasks::value_impl]
 impl ModuleIdStrategy for GlobalModuleIdStrategy {
     #[turbo_tasks::function]
-    async fn get_module_id(&self, ident: Vc<AssetIdent>) -> Result<Vc<ModuleId>> {
-        let ident_string = ident.to_string().await?.clone_value();
-        if let Some(module_id) = self.module_id_map.get(&ident_string) {
-            return Ok(module_id.clone().cell());
+    async fn get_module_id(&self, ident: ResolvedVc<AssetIdent>) -> Result<Vc<ModuleId>> {
+        if let Some(module_id) = self.module_id_map.get(&ident) {
+            const JS_MAX_SAFE_INTEGER: u64 = (1u64 << 53) - 1;
+            if *module_id > JS_MAX_SAFE_INTEGER {
+                bail!("Numeric module id is too large: {}", module_id);
+            }
+            return Ok(ModuleId::Number(*module_id).cell());
         }
+
+        let ident_string = ident.to_string().await?;
+        if !ident_string.ends_with("[app-client] (ecmascript, next/dynamic entry)") {
+            // TODO: This shouldn't happen, but is a temporary workaround to ignore next/dynamic
+            // imports of a server component from another server component.
+
+            ModuleIssue {
+                ident,
+                title: StyledString::Text(
+                    format!("ModuleId not found for ident: {:?}", ident_string).into(),
+                )
+                .resolved_cell(),
+                description: StyledString::Text(
+                    format!("ModuleId not found for ident: {:?}", ident_string).into(),
+                )
+                .resolved_cell(),
+            }
+            .resolved_cell()
+            .emit();
+        }
+
         Ok(ModuleId::String(
             hash_xxh3_hash64(ident.to_string().await?)
                 .to_string()