Apply the next/dynamic SWC transform (vercel#3184)

This PR adds the next/dynamic SWC transform to the repo. This is copied
over from the implementation in
[](https://github.com/vercel/next.js/blob/3e2e39c55939bd156cbb4dab926aaaa126b53d1c/packages/next-swc/crates/core/src/next_dynamic.rs),
with an additional mode for Turbopack.

For now, it applies it to all sources files (excluding node_modules, but
including embedded modules).

I've refactored the Next transforms logic to make it more easily
extendable in the future, and avoid creating more Vcs than absolutely
necessary.

This PR is the last item in the list of what's needed to support
next/dynamic in development.

.prettierignore

@@ -20,4 +20,5 @@ crates/turbopack-node/js/src/compiled
 crates/turbopack/bench.json
 # This file has intentional trailing commas
 crates/turbopack/tests/node-file-trace/integration/ts-package/tsconfig.json
-crates/next-transform-strip-page-exports/tests/fixtures
+crates/next-transform-strip-page-exports/tests
+crates/next-transform-dynamic/tests

Cargo.toml

@@ -6,6 +6,7 @@ members = [
   "crates/next-binding",
   "crates/next-core",
   "crates/next-dev",
+  "crates/next-transform-dynamic",
   "crates/next-transform-strip-page-exports",
   "crates/node-file-trace",
   "crates/swc-ast-explorer",
@@ -45,6 +46,8 @@ default-members = [
   "crates/next-binding",
   "crates/next-core",
   "crates/next-dev",
+  "crates/next-transform-dynamic",
+  "crates/next-transform-strip-page-exports",
   "crates/node-file-trace",
   "crates/swc-ast-explorer",
   "crates/turbo-malloc",

crates/next-core/js/src/entry/server-renderer.tsx

@@ -9,6 +9,7 @@ import type { IncomingMessage, ServerResponse } from "node:http";
 
 import { renderToHTML, RenderOpts } from "next/dist/server/render";
 import type { BuildManifest } from "next/dist/server/get-page-files";
+import type { ReactLoadableManifest } from "next/dist/server/load-components";
 
 import { ServerResponseShim } from "@vercel/turbopack-next/internal/http";
 import type { Ipc } from "@vercel/turbopack-next/ipc/index";
@@ -120,7 +121,7 @@ async function runOperation(
     Document,
     pageConfig: {},
     buildManifest,
-    reactLoadableManifest: {},
+    reactLoadableManifest: createReactLoadableManifestProxy(),
     ComponentMod: {
       default: comp,
       ...otherExports,
@@ -208,3 +209,52 @@ async function runOperation(
     pageData: pageData,
   };
 }
+
+type ManifestItem = {
+  id: string;
+  chunks: string[];
+};
+
+/**
+ * During compilation, Next.js builds a manifest of dynamic imports with the
+ * `ReactLoadablePlugin` for webpack.
+ *
+ * At the same time, the next/dynamic transform converts each `dynamic()` call
+ * so it contains a key to the corresponding entry within that manifest.
+ *
+ * During server-side rendering, each `dynamic()` call will be recorded and its
+ * corresponding entry in the manifest will be looked up.
+ * * The entry's chunks will be asynchronously loaded on the client using a
+ *   <script defer> tag.
+ * * The entry's module id will be appended to a list of dynamic module ids.
+ *
+ * On the client-side, during hydration, the dynamic module ids are used to
+ * initialize the corresponding <Loadable> components.
+ *
+ * In development, Turbopack works differently: instead of building a static
+ * manifest, each `dynamic()` call will embed its own manifest entry within a
+ * serialized string key. Hence the need for a proxy that can dynamically
+ * deserialize the manifest entries from that string key.
+ */
+function createReactLoadableManifestProxy(): ReactLoadableManifest {
+  return new Proxy(
+    {},
+    {
+      get: (_target, prop: string, _receiver) => {
+        const { id, chunks } = JSON.parse(prop) as ManifestItem;
+
+        return {
+          id,
+          files: chunks.map((chunk) => {
+            // Turbopack prefixes chunks with "_next/", but Next.js expects
+            // them to be relative to the build directory.
+            if (chunk.startsWith("_next/")) {
+              return chunk.slice("_next/".length);
+            }
+            return chunk;
+          }),
+        };
+      },
+    }
+  );
+}

crates/next-core/src/app_source.rs

@@ -54,8 +54,8 @@ use crate::{
         },
         transition::NextClientTransition,
     },
+    next_client_chunks::client_chunks_transition::NextClientChunksTransitionVc,
     next_client_component::{
-        client_chunks_transition::NextClientChunksTransition,
         server_to_client_transition::NextServerToClientTransition,
         ssr_client_module_transition::NextSSRClientModuleTransition,
     },
@@ -67,42 +67,6 @@ use crate::{
     util::{pathname_for_path, regular_expression_for_path},
 };
 
-#[turbo_tasks::function]
-fn next_client_chunks_transition(
-    project_path: FileSystemPathVc,
-    execution_context: ExecutionContextVc,
-    app_dir: FileSystemPathVc,
-    server_root: FileSystemPathVc,
-    browserslist_query: &str,
-    next_config: NextConfigVc,
-) -> TransitionVc {
-    let ty = Value::new(ClientContextType::App { app_dir });
-    let client_environment = get_client_environment(browserslist_query);
-    let client_chunking_context =
-        get_client_chunking_context(project_path, server_root, client_environment, ty);
-
-    let client_module_options_context = get_client_module_options_context(
-        project_path,
-        execution_context,
-        client_environment,
-        ty,
-        next_config,
-    );
-    NextClientChunksTransition {
-        client_chunking_context,
-        client_module_options_context,
-        client_resolve_options_context: get_client_resolve_options_context(
-            project_path,
-            ty,
-            next_config,
-        ),
-        client_environment,
-        server_root,
-    }
-    .cell()
-    .into()
-}
-
 #[turbo_tasks::function]
 async fn next_client_transition(
     project_path: FileSystemPathVc,
@@ -240,16 +204,18 @@ fn app_context(
             next_config,
         ),
     );
+    let client_ty = Value::new(ClientContextType::App { app_dir });
     transitions.insert(
         "next-client-chunks".to_string(),
-        next_client_chunks_transition(
+        NextClientChunksTransitionVc::new(
             project_path,
             execution_context,
-            app_dir,
+            client_ty,
             server_root,
             browserslist_query,
             next_config,
-        ),
+        )
+        .into(),
     );
     transitions.insert(
         "next-ssr-client-module".to_string(),

crates/next-core/src/lib.rs

@@ -9,6 +9,7 @@ mod fallback;
 pub mod manifest;
 mod next_build;
 pub mod next_client;
+mod next_client_chunks;
 mod next_client_component;
 pub mod next_config;
 mod next_font_google;

crates/next-core/src/next_client/context.rs

@@ -2,13 +2,13 @@ use core::{default::Default, result::Result::Ok};
 use std::collections::HashMap;
 
 use anyhow::Result;
-use turbo_tasks::{primitives::StringsVc, Value};
+use turbo_tasks::Value;
 use turbo_tasks_env::ProcessEnvVc;
 use turbo_tasks_fs::FileSystemPathVc;
 use turbopack::{
     module_options::{
         module_options_context::{ModuleOptionsContext, ModuleOptionsContextVc},
-        ModuleRule, ModuleRuleCondition, ModuleRuleEffect, PostCssTransformOptions,
+        PostCssTransformOptions,
     },
     resolve_options_context::{ResolveOptionsContext, ResolveOptionsContextVc},
     transition::TransitionsByNameVc,
@@ -18,13 +18,12 @@ use turbopack_core::{
     chunk::{dev::DevChunkingContextVc, ChunkingContextVc},
     context::AssetContextVc,
     environment::{BrowserEnvironment, EnvironmentIntention, EnvironmentVc, ExecutionEnvironment},
-    reference_type::{ReferenceType, UrlReferenceSubType},
     resolve::{parse::RequestVc, pattern::Pattern},
 };
-use turbopack_ecmascript::{EcmascriptInputTransform, EcmascriptInputTransformsVc};
 use turbopack_env::ProcessEnvAssetVc;
 use turbopack_node::execution_context::ExecutionContextVc;
 
+use super::transforms::get_next_client_transforms_rules;
 use crate::{
     embed_js::attached_next_js_package_path,
     env::env_for_js,
@@ -103,6 +102,7 @@ pub async fn get_client_module_options_context(
     ty: Value<ClientContextType>,
     next_config: NextConfigVc,
 ) -> Result<ModuleOptionsContextVc> {
+    let custom_rules = get_next_client_transforms_rules(ty.into_value()).await?;
     let resolve_options_context = get_client_resolve_options_context(project_path, ty, next_config);
     let enable_react_refresh =
         assert_can_resolve_react_refresh(project_path, resolve_options_context)
@@ -133,42 +133,12 @@ pub async fn get_client_module_options_context(
             foreign_code_context_condition(next_config).await?,
             module_options_context.clone().cell(),
         )],
+        custom_rules,
         ..module_options_context
-    };
-
-    Ok(add_next_font_transform(module_options_context.cell()))
-}
-
-#[turbo_tasks::function]
-pub async fn add_next_font_transform(
-    module_options_context: ModuleOptionsContextVc,
-) -> Result<ModuleOptionsContextVc> {
-    #[allow(unused_mut)] // This is mutated when next-font-local is enabled
-    let mut font_loaders = vec!["@next/font/google".to_owned()];
-    #[cfg(feature = "next-font-local")]
-    font_loaders.push("@next/font/local".to_owned());
+    }
+    .cell();
 
-    let mut module_options_context = module_options_context.await?.clone_value();
-    module_options_context.custom_rules.push(ModuleRule::new(
-        // TODO: Only match in pages (not pages/api), app/, etc.
-        ModuleRuleCondition::all(vec![
-            ModuleRuleCondition::not(ModuleRuleCondition::ReferenceType(ReferenceType::Url(
-                UrlReferenceSubType::Undefined,
-            ))),
-            ModuleRuleCondition::any(vec![
-                ModuleRuleCondition::ResourcePathEndsWith(".js".to_string()),
-                ModuleRuleCondition::ResourcePathEndsWith(".jsx".to_string()),
-                ModuleRuleCondition::ResourcePathEndsWith(".ts".to_string()),
-                ModuleRuleCondition::ResourcePathEndsWith(".tsx".to_string()),
-            ]),
-        ]),
-        vec![ModuleRuleEffect::AddEcmascriptTransforms(
-            EcmascriptInputTransformsVc::cell(vec![EcmascriptInputTransform::NextJsFont(
-                StringsVc::cell(font_loaders),
-            )]),
-        )],
-    ));
-    Ok(module_options_context.cell())
+    Ok(module_options_context)
 }
 
 #[turbo_tasks::function]

crates/next-core/src/next_client/mod.rs

@@ -1,3 +1,4 @@
 pub(crate) mod context;
 pub(crate) mod runtime_entry;
+pub(crate) mod transforms;
 pub(crate) mod transition;

crates/next-core/src/next_client/transforms.rs

@@ -0,0 +1,40 @@
+use anyhow::Result;
+use turbopack::module_options::ModuleRule;
+use turbopack_ecmascript::NextJsPageExportFilter;
+
+use crate::{
+    next_client::context::ClientContextType,
+    next_shared::transforms::{
+        get_next_dynamic_transform_rule, get_next_font_transform_rule,
+        get_next_pages_transforms_rule,
+    },
+};
+
+/// Returns a list of module rules which apply client-side, Next.js-specific
+/// transforms.
+pub async fn get_next_client_transforms_rules(
+    context_ty: ClientContextType,
+) -> Result<Vec<ModuleRule>> {
+    let mut rules = vec![];
+
+    rules.push(get_next_font_transform_rule());
+
+    let pages_dir = match context_ty {
+        ClientContextType::Pages { pages_dir } => {
+            rules.push(
+                get_next_pages_transforms_rule(pages_dir, NextJsPageExportFilter::StripDataExports)
+                    .await?,
+            );
+            Some(pages_dir)
+        }
+        ClientContextType::App { .. } | ClientContextType::Fallback | ClientContextType::Other => {
+            None
+        }
+    };
+
+    rules.push(get_next_dynamic_transform_rule(
+        true, false, false, pages_dir,
+    ));
+
+    Ok(rules)
+}

crates/next-core/src/next_client_component/client_chunks_transition.rs → crates/next-core/src/next_client_chunks/client_chunks_transition.rs

@@ -1,4 +1,5 @@
 use anyhow::Result;
+use turbo_tasks::Value;
 use turbo_tasks_fs::FileSystemPathVc;
 use turbopack::{
     ecmascript::chunk::EcmascriptChunkPlaceableVc,
@@ -8,8 +9,16 @@ use turbopack::{
     ModuleAssetContextVc,
 };
 use turbopack_core::{asset::AssetVc, chunk::ChunkingContextVc, environment::EnvironmentVc};
+use turbopack_node::execution_context::ExecutionContextVc;
 
 use super::with_chunks::WithChunksAsset;
+use crate::{
+    next_client::context::{
+        get_client_chunking_context, get_client_environment, get_client_module_options_context,
+        get_client_resolve_options_context, ClientContextType,
+    },
+    next_config::NextConfigVc,
+};
 
 #[turbo_tasks::value(shared)]
 pub struct NextClientChunksTransition {
@@ -20,6 +29,43 @@ pub struct NextClientChunksTransition {
     pub server_root: FileSystemPathVc,
 }
 
+#[turbo_tasks::value_impl]
+impl NextClientChunksTransitionVc {
+    #[turbo_tasks::function]
+    pub fn new(
+        project_path: FileSystemPathVc,
+        execution_context: ExecutionContextVc,
+        ty: Value<ClientContextType>,
+        server_root: FileSystemPathVc,
+        browserslist_query: &str,
+        next_config: NextConfigVc,
+    ) -> NextClientChunksTransitionVc {
+        let client_environment = get_client_environment(browserslist_query);
+        let client_chunking_context =
+            get_client_chunking_context(project_path, server_root, client_environment, ty);
+
+        let client_module_options_context = get_client_module_options_context(
+            project_path,
+            execution_context,
+            client_environment,
+            ty,
+            next_config,
+        );
+        NextClientChunksTransition {
+            client_chunking_context,
+            client_module_options_context,
+            client_resolve_options_context: get_client_resolve_options_context(
+                project_path,
+                ty,
+                next_config,
+            ),
+            client_environment,
+            server_root,
+        }
+        .cell()
+    }
+}
+
 #[turbo_tasks::value_impl]
 impl Transition for NextClientChunksTransition {
     #[turbo_tasks::function]