Turbopack: error when importing Typescript in node_modules (#83990)

Webpack treats TS files in node_modules as Javascript files (i.e. you get a JS parse error)

Turbopack previously did a similar thing (though it parsed as Typescript, but then never ran the TS AST transform).
This never worked properly, it failed due to some unreachable assertion in the swc transforms.

Now it fails with a proper error ~~, though too many of them~~:

```
./node_modules/.pnpm/pkg@file+pkg/node_modules/pkg/index.ts
Missing module type
The module type effect must be applied before adding Ecmascript transforms


./node_modules/.pnpm/pkg@file+pkg/node_modules/pkg/index.ts
Unknown module type
This module doesn't have an associated type. Use a known file extension, or register a loader for it.

Read more: https://nextjs.org/docs/app/api-reference/next-config-js/turbo#webpack-loaders
    at ignore-listed frames
```

Additionally: don't run styled-jsx/styled-components/relay/... in node_modules in the server. This aligns it what we already had configured for the client context

Author: mischnic

crates/next-core/src/next_client/transforms.rs

@@ -32,7 +32,10 @@ pub async fn get_next_client_transforms_rules(
 
     let modularize_imports_config = &next_config.modularize_imports().await?;
     let enable_mdx_rs = next_config.mdx_rs().await?.is_some();
-    rules.push(get_next_lint_transform_rule(enable_mdx_rs));
+
+    if !foreign_code {
+        rules.push(get_next_lint_transform_rule(enable_mdx_rs));
+    }
 
     if !modularize_imports_config.is_empty() {
         rules.push(get_next_modularize_imports_rule(

crates/next-core/src/next_server/context.rs

@@ -559,14 +559,16 @@ pub async fn get_server_module_options_context(
     .flatten()
     .collect();
 
-    // Custom ecma transform rules selectively being applied depends on the server
-    // context type.
-    let styled_components_transform_rule =
-        get_styled_components_transform_rule(next_config).await?;
-    // It's important the client's browserlist config is used for styled-jsx, otherwise we transpile
-    // the CSS to be compatible with Node.js 20.
-    let styled_jsx_transform_rule =
-        get_styled_jsx_transform_rule(next_config, client_environment.runtime_versions()).await?;
+    // Only relevant for pages, not routes/etc.
+    let page_transform_rules: Vec<ModuleRule> = vec![
+        get_styled_components_transform_rule(next_config).await?,
+        // It's important the client's browserlist config is used for styled-jsx, otherwise we
+        // transpile the CSS to be compatible with Node.js 20.
+        get_styled_jsx_transform_rule(next_config, client_environment.runtime_versions()).await?,
+    ]
+    .into_iter()
+    .flatten()
+    .collect();
 
     let source_maps = if *next_config.server_source_maps().await? {
         SourceMapsType::Full
@@ -608,23 +610,16 @@ pub async fn get_server_module_options_context(
 
     let module_options_context = match ty {
         ServerContextType::Pages { .. } | ServerContextType::PagesApi { .. } => {
-            let mut custom_source_transform_rules: Vec<ModuleRule> =
-                vec![styled_components_transform_rule, styled_jsx_transform_rule]
-                    .into_iter()
-                    .flatten()
-                    .collect();
-
+            next_server_rules.extend(page_transform_rules);
             if let ServerContextType::Pages { .. } = ty {
-                custom_source_transform_rules.push(
+                next_server_rules.push(
                     get_next_react_server_components_transform_rule(next_config, false, None)
                         .await?,
                 );
             }
 
-            next_server_rules.extend(custom_source_transform_rules.iter().cloned());
             next_server_rules.extend(source_transform_rules);
 
-            foreign_next_server_rules.extend(custom_source_transform_rules);
             foreign_next_server_rules.extend(internal_custom_rules);
 
             let url_rewrite_behavior = Some(
@@ -690,21 +685,13 @@ pub async fn get_server_module_options_context(
             }
         }
         ServerContextType::AppSSR { app_dir, .. } => {
-            let mut custom_source_transform_rules: Vec<ModuleRule> =
-                vec![styled_components_transform_rule, styled_jsx_transform_rule]
-                    .into_iter()
-                    .flatten()
-                    .collect();
-
-            foreign_next_server_rules.extend(custom_source_transform_rules.iter().cloned());
             foreign_next_server_rules.extend(internal_custom_rules);
 
-            custom_source_transform_rules.push(
+            next_server_rules.extend(page_transform_rules.clone());
+            next_server_rules.push(
                 get_next_react_server_components_transform_rule(next_config, false, Some(app_dir))
                     .await?,
             );
-
-            next_server_rules.extend(custom_source_transform_rules.clone());
             next_server_rules.extend(source_transform_rules);
 
             let foreign_code_module_options_context = ModuleOptionsContext {
@@ -755,33 +742,29 @@ pub async fn get_server_module_options_context(
             ecmascript_client_reference_transition_name,
             ..
         } => {
-            let mut custom_source_transform_rules: Vec<ModuleRule> =
-                vec![styled_components_transform_rule, styled_jsx_transform_rule]
-                    .into_iter()
-                    .flatten()
-                    .collect();
-
-            if let Some(ecmascript_client_reference_transition_name) =
-                ecmascript_client_reference_transition_name
-            {
-                custom_source_transform_rules.push(get_ecma_transform_rule(
-                    Box::new(ClientDirectiveTransformer::new(
-                        ecmascript_client_reference_transition_name,
-                    )),
-                    enable_mdx_rs.is_some(),
-                    EcmascriptTransformStage::Preprocess,
-                ));
-            }
+            next_server_rules.extend(page_transform_rules);
+
+            let client_directive_transformer = ecmascript_client_reference_transition_name.map(
+                |ecmascript_client_reference_transition_name| {
+                    get_ecma_transform_rule(
+                        Box::new(ClientDirectiveTransformer::new(
+                            ecmascript_client_reference_transition_name,
+                        )),
+                        enable_mdx_rs.is_some(),
+                        EcmascriptTransformStage::Preprocess,
+                    )
+                },
+            );
 
-            foreign_next_server_rules.extend(custom_source_transform_rules.iter().cloned());
+            foreign_next_server_rules.extend(client_directive_transformer.clone());
             foreign_next_server_rules.extend(internal_custom_rules);
 
-            custom_source_transform_rules.push(
+            next_server_rules.extend(client_directive_transformer.clone());
+            next_server_rules.push(
                 get_next_react_server_components_transform_rule(next_config, true, Some(app_dir))
                     .await?,
             );
 
-            next_server_rules.extend(custom_source_transform_rules.clone());
             next_server_rules.extend(source_transform_rules);
 
             let foreign_code_module_options_context = ModuleOptionsContext {
@@ -909,35 +892,28 @@ pub async fn get_server_module_options_context(
             app_dir,
             ecmascript_client_reference_transition_name,
         } => {
-            let mut custom_source_transform_rules: Vec<ModuleRule> =
-                vec![styled_components_transform_rule, styled_jsx_transform_rule]
-                    .into_iter()
-                    .flatten()
-                    .collect();
-
-            if let Some(ecmascript_client_reference_transition_name) =
-                ecmascript_client_reference_transition_name
-            {
-                custom_source_transform_rules.push(get_ecma_transform_rule(
-                    Box::new(ClientDirectiveTransformer::new(
-                        ecmascript_client_reference_transition_name,
-                    )),
-                    enable_mdx_rs.is_some(),
-                    EcmascriptTransformStage::Preprocess,
-                ));
-            } else {
-                custom_source_transform_rules.push(get_ecma_transform_rule(
-                    Box::new(ClientDisallowedDirectiveTransformer::new(
-                        "next/dist/client/use-client-disallowed.js".to_string(),
-                    )),
-                    enable_mdx_rs.is_some(),
-                    EcmascriptTransformStage::Preprocess,
-                ));
-            }
-
-            custom_source_transform_rules.push(
+            let custom_source_transform_rules: Vec<ModuleRule> = vec![
+                if let Some(ecmascript_client_reference_transition_name) =
+                    ecmascript_client_reference_transition_name
+                {
+                    get_ecma_transform_rule(
+                        Box::new(ClientDirectiveTransformer::new(
+                            ecmascript_client_reference_transition_name,
+                        )),
+                        enable_mdx_rs.is_some(),
+                        EcmascriptTransformStage::Preprocess,
+                    )
+                } else {
+                    get_ecma_transform_rule(
+                        Box::new(ClientDisallowedDirectiveTransformer::new(
+                            "next/dist/client/use-client-disallowed.js".to_string(),
+                        )),
+                        enable_mdx_rs.is_some(),
+                        EcmascriptTransformStage::Preprocess,
+                    )
+                },
                 get_next_react_server_components_transform_rule(next_config, true, app_dir).await?,
-            );
+            ];
 
             internal_custom_rules.extend(custom_source_transform_rules.iter().cloned());
 

crates/next-core/src/next_server/transforms.rs

@@ -39,7 +39,9 @@ pub async fn get_next_server_transforms_rules(
     let modularize_imports_config = &next_config.modularize_imports().await?;
     let mdx_rs = next_config.mdx_rs().await?.is_some();
 
-    rules.push(get_next_lint_transform_rule(mdx_rs));
+    if !foreign_code {
+        rules.push(get_next_lint_transform_rule(mdx_rs));
+    }
 
     if !modularize_imports_config.is_empty() {
         rules.push(get_next_modularize_imports_rule(

test/e2e/app-dir/app-routes-client-component/node_modules/my-module/MyModuleClientComponent.tsx renamed to test/e2e/app-dir/app-routes-client-component/node_modules/my-module/MyModuleClientComponent.js

@@ -0,0 +1,10 @@
+import * as React from 'react'
+
+export default function Root({ children }) {
+  return (
+    <html>
+      <head></head>
+      <body>{children}</body>
+    </html>
+  )
+}

test/e2e/transpile-packages-typescript-foreign/app/layout.js

@@ -0,0 +1,5 @@
+import { Foo } from 'pkg/index.ts'
+
+export default function Page() {
+  return <main>Hello {Foo}</main>
+}

test/e2e/transpile-packages-typescript-foreign/app/page.js

@@ -0,0 +1,62 @@
+import { nextTestSetup } from 'e2e-utils'
+
+describe('transpile-packages-typescript-foreign', () => {
+  describe('without transpilePackages', () => {
+    const { next, skipped } = nextTestSetup({
+      files: __dirname,
+      skipDeployment: true,
+      skipStart: true,
+      dependencies: {
+        pkg: `file:./pkg`,
+      },
+    })
+
+    if (skipped) {
+      return
+    }
+
+    it('should fail', async () => {
+      try {
+        await next.start()
+        await next.render('/')
+      } catch (e) {}
+
+      if (process.env.IS_TURBOPACK_TEST) {
+        expect(next.cliOutput).toContain(`pkg/index.ts
+Unknown module type
+This module doesn't have an associated type`)
+        expect(
+          next.cliOutput.match(/Unknown module type/g).length
+        ).toBeLessThanOrEqual(1)
+        expect(
+          next.cliOutput.match(/Missing module type/g).length
+        ).toBeLessThanOrEqual(1)
+      } else {
+        expect(next.cliOutput).toContain(`pkg/index.ts
+Module parse failed: Unexpected token`)
+      }
+    })
+  })
+
+  describe('with transpilePackages', () => {
+    const { next, skipped } = nextTestSetup({
+      files: __dirname,
+      skipDeployment: true,
+      dependencies: {
+        pkg: `file:./pkg`,
+      },
+      nextConfig: {
+        transpilePackages: ['pkg'],
+      },
+    })
+
+    if (skipped) {
+      return
+    }
+
+    it('should work', async () => {
+      const $ = await next.render$('/')
+      expect($('main').text()).toEqual('Hello 123')
+    })
+  })
+})

test/e2e/transpile-packages-typescript-foreign/index.test.ts

@@ -0,0 +1 @@
+export type Cents = number

test/e2e/transpile-packages-typescript-foreign/pkg/cents.ts

@@ -0,0 +1,5 @@
+import type { Cents } from './cents'
+
+export type Type = Cents
+
+export const Foo = 123

test/e2e/transpile-packages-typescript-foreign/pkg/index.ts

@@ -0,0 +1,4 @@
+{
+  "name": "pkg",
+  "version": "1.0.0"
+}