Support HMR Update Source Maps (vercel#371)

This allows us to reference the source maps for individual chunk items when delivering them for HMR updates.

We currently send down the update via a `JSON.stringify()` of the updated module's code. We then `eval` this code to generate the new "module", and call registered hot handlers with the new module to update their scopes. Thankfully, we can include a `sourceMappingUrl` comment in the evaled code, so we can get away with deferring creating the source map's content until dev tools is opened.

This is a first pass, it really needs to be cleaned up. But at least it's working.


Co-authored-by: Tobias Koppers <[email protected]>

.gitattributes

@@ -1 +1 @@
-crates/turbopack/tests/snapshot/*/output/ linguist-generated=true
+crates/turbopack/tests/snapshot/*/output/** linguist-generated=true

crates/turbopack-core/src/code_builder.rs

@@ -1,4 +1,7 @@
-use std::{fmt::Write as _, ops};
+use std::{
+    fmt::{Result as FmtResult, Write},
+    ops,
+};
 
 use anyhow::Result;
 use turbo_tasks::primitives::StringVc;
@@ -38,8 +41,8 @@ impl Code {
 
     /// Setting breakpoints on synthetic code can cause weird behaviors
     /// because Chrome will treat the location as belonging to the previous
-    /// original code section. By inserting an empty source map when reaching an
-    /// synethic section directly after an original section, we tell Chrome
+    /// original code section. By inserting an empty source map when reaching a
+    /// synthetic section directly after an original section, we tell Chrome
     /// that the previous map ended at this point.
     fn push_map(&mut self, map: Option<EncodedSourceMapVc>) {
         if map.is_none() && matches!(self.mappings.last(), None | Some((_, None))) {
@@ -53,7 +56,7 @@ impl Code {
     /// Pushes synthetic runtime code without an associated source map. This is
     /// the default concatenation operation, but it's designed to be used
     /// with the `+=` operator.
-    pub fn push_str(&mut self, code: &str) {
+    fn push_str(&mut self, code: &str) {
         self.push_source(code, None);
     }
 
@@ -69,7 +72,7 @@ impl Code {
     /// into this instance.
     pub fn push_code(&mut self, prebuilt: &Code) {
         if let Some((index, map)) = prebuilt.mappings.first() {
-            debug_assert!(matches!(map, Some(_)), "the first mapping is never a None");
+            debug_assert!(map.is_some(), "the first mapping is never a None");
 
             if *index > 0 {
                 // If the index is positive, then the code starts with a synthetic section. We
@@ -104,14 +107,8 @@ impl ops::AddAssign<&str> for Code {
     }
 }
 
-impl ops::AddAssign<String> for Code {
-    fn add_assign(&mut self, rhs: String) {
-        self.push_str(&rhs);
-    }
-}
-
-impl std::fmt::Write for Code {
-    fn write_str(&mut self, s: &str) -> std::fmt::Result {
+impl Write for Code {
+    fn write_str(&mut self, s: &str) -> FmtResult {
         self.push_str(s);
         Ok(())
     }

crates/turbopack-core/src/source_map.rs

@@ -15,21 +15,30 @@ pub struct SourceMapAsset {
     /// File path of the associated file (not the map's path, the file's path).
     path: FileSystemPathVc,
 
+    /// A version fingerprint so that the associated file can point to a unique
+    /// source map.
+    version: StringVc,
+
     /// The code from which we can generate a source map.
     code: CodeVc,
 }
 
 impl SourceMapAssetVc {
-    pub fn new(path: FileSystemPathVc, code: CodeVc) -> Self {
-        SourceMapAsset { path, code }.cell()
+    pub fn new(path: FileSystemPathVc, version: StringVc, code: CodeVc) -> Self {
+        SourceMapAsset {
+            path,
+            version,
+            code,
+        }
+        .cell()
     }
 }
 
 #[turbo_tasks::value_impl]
 impl Asset for SourceMapAsset {
     #[turbo_tasks::function]
-    fn path(&self) -> FileSystemPathVc {
-        self.path.append(".map")
+    async fn path(&self) -> Result<FileSystemPathVc> {
+        Ok(self.path.append(&format!(".{}.map", self.version.await?)))
     }
 
     #[turbo_tasks::function]
@@ -44,8 +53,8 @@ impl Asset for SourceMapAsset {
     }
 }
 
-/// A reference to a SourceMapAsset, used to inform the dev server/build system
-/// of the presense of t the source map
+/// A reference to a [`SourceMapAsset`], used to inform the dev server/build
+/// system of the presence of the source map
 #[turbo_tasks::value]
 pub struct SourceMapAssetReference {
     asset: SourceMapAssetVc,

crates/turbopack-ecmascript/src/chunk/mod.rs

@@ -312,8 +312,8 @@ async fn module_factory(content: EcmascriptChunkItemContentVc) -> Result<CodeVc>
 
 #[derive(Serialize)]
 struct EcmascriptChunkUpdate<'a> {
-    added: HashMap<&'a ModuleId, &'a str>,
-    modified: HashMap<&'a ModuleId, &'a str>,
+    added: HashMap<&'a ModuleId, HmrUpdateEntry<'a>>,
+    modified: HashMap<&'a ModuleId, HmrUpdateEntry<'a>>,
     deleted: HashSet<&'a ModuleId>,
 }
 
@@ -398,7 +398,12 @@ impl EcmascriptChunkContentVc {
 
         if code.has_source_map() {
             let filename = chunk_path.file_name().unwrap();
-            write!(code, "\n\n//# sourceMappingURL={}.map", filename)?;
+            let version = self.version().as_version().id().await?;
+            write!(
+                code,
+                "\n\n//# sourceMappingURL={}.{}.map",
+                filename, version
+            )?;
         }
 
         Ok(code.cell())
@@ -459,7 +464,17 @@ impl VersionedContent for EcmascriptChunkContent {
             let id = &**id;
             if let Some(entry) = module_factories.remove(id) {
                 if entry.hash != *hash {
-                    modified.insert(id, entry.source_code());
+                    modified.insert(
+                        id,
+                        HmrUpdateEntry {
+                            code: entry.source_code(),
+                            map: format!(
+                                "{}.{}.map",
+                                this.chunk_path.await?.path,
+                                encode_hex(entry.hash)
+                            ),
+                        },
+                    );
                 }
             } else {
                 deleted.insert(id);
@@ -468,7 +483,17 @@ impl VersionedContent for EcmascriptChunkContent {
 
         // Remaining entries are added
         for (id, entry) in module_factories {
-            added.insert(id, entry.source_code());
+            added.insert(
+                id,
+                HmrUpdateEntry {
+                    code: entry.source_code(),
+                    map: format!(
+                        "{}.{}.map",
+                        this.chunk_path.await?.path,
+                        encode_hex(entry.hash)
+                    ),
+                },
+            );
         }
 
         let update = if added.is_empty() && modified.is_empty() && deleted.is_empty() {
@@ -490,6 +515,12 @@ impl VersionedContent for EcmascriptChunkContent {
     }
 }
 
+#[derive(serde::Serialize)]
+struct HmrUpdateEntry<'a> {
+    code: &'a str,
+    map: String,
+}
+
 #[turbo_tasks::value(serialization = "none")]
 struct EcmascriptChunkVersion {
     module_factories_hashes: HashMap<ModuleIdReadRef, u64>,
@@ -659,13 +690,33 @@ impl Asset for EcmascriptChunk {
             references.push(ChunkGroupReferenceVc::new(*chunk_group).into());
         }
 
+        let chunk_content = self_vc.chunk_content();
         references.push(
             SourceMapAssetReferenceVc::new(SourceMapAssetVc::new(
                 self_vc.path(),
-                self_vc.chunk_content().code(),
+                chunk_content.as_versioned_content().version().id(),
+                chunk_content.code(),
             ))
             .into(),
         );
+
+        let chunk_items = content.chunk_items.await?;
+        for item in chunk_items.into_iter() {
+            let content = item.content(chunk_context(this.context), this.context);
+            let code = module_factory(content);
+            if code.await?.has_source_map() {
+                references.push(
+                    SourceMapAssetReferenceVc::new(SourceMapAssetVc::new(
+                        self_vc.path(),
+                        StringVc::cell(encode_hex(hash_xxh3_hash64(
+                            code.source_code().await?.as_bytes(),
+                        ))),
+                        code,
+                    ))
+                    .into(),
+                );
+            }
+        }
         Ok(AssetReferencesVc::cell(references))
     }
 

crates/turbopack-ecmascript/src/chunk/runtime.js

@@ -305,17 +305,20 @@
     return `Dependency chain: ${dependencyChain.join(' -> ')}`
   }
 
+  function _eval(factory) {
+    const code = `${factory.code}\n\n//# sourceMappingURL=${factory.map}`
+    return eval(code)
+  }
+
   function computeOutdatedModules(update) {
     const outdatedModules = new Set()
     const newModuleFactories = new Map()
 
-    for (const [moduleId, moduleFactoryStr] of Object.entries(update.added)) {
-      newModuleFactories.set(moduleId, eval(moduleFactoryStr))
+    for (const [moduleId, factory] of Object.entries(update.added)) {
+      newModuleFactories.set(moduleId, _eval(factory))
     }
 
-    for (const [moduleId, moduleFactoryStr] of Object.entries(
-      update.modified,
-    )) {
+    for (const [moduleId, factory] of Object.entries(update.modified)) {
       const effect = getAffectedModuleEffects(moduleId)
 
       switch (effect.type) {
@@ -332,7 +335,7 @@
             )}.`,
           )
         case 'accepted':
-          newModuleFactories.set(moduleId, eval(moduleFactoryStr))
+          newModuleFactories.set(moduleId, _eval(factory))
           for (const outdatedModuleId of effect.outdatedModules) {
             outdatedModules.add(outdatedModuleId)
           }

crates/turbopack/tests/snapshot/example/async_chunk/output/snapshot_example_async_chunk_input_index.js.d48f51079f679975.map

@@ -0,0 +1,6 @@
+{
+  "version": 3,
+  "sections": [
+    {"offset": {"line": 2, "column": 0}, "map": {"version":3,"sources":["/[project]/snapshot/example/async_chunk/input/index.js"],"sourcesContent":["import('foo').then(({ foo }) => {\n  foo(true)\n})\n"],"names":[],"mappings":"AAAA,wIAAa,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,CAAA,EAAE,GAAK;IAC9B,GAAG,CAAC,IAAI,CAAC;AACX,CAAC,CAAC"}},
+    {"offset": {"line": 5, "column": 0}, "map": {"version": 3, "names": [], "sources": [], "mappings": "A"}}]
+}

crates/turbopack/tests/snapshot/example/async_chunk/output/snapshot_example_async_chunk_input_node_modules_foo_index.js.cf8ba911a655ce0a.map

@@ -0,0 +1,6 @@
+{
+  "version": 3,
+  "sections": [
+    {"offset": {"line": 2, "column": 0}, "map": {"version":3,"sources":["/[project]/snapshot/example/async_chunk/input/node_modules/foo/index.js"],"sourcesContent":["export function foo(value) {\n  console.assert(value);\n}\n"],"names":[],"mappings":"AAAA;;;AAAO,SAAS,GAAG,CAAC,KAAK,EAAE;IACzB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACxB,CAAC"}},
+    {"offset": {"line": 8, "column": 0}, "map": {"version": 3, "names": [], "sources": [], "mappings": "A"}}]
+}

crates/turbopack/tests/snapshot/example/chunked/output/snapshot_example_chunked_input_index.js.c6c2def1c0038195.map

@@ -0,0 +1,6 @@
+{
+  "version": 3,
+  "sections": [
+    {"offset": {"line": 2, "column": 0}, "map": {"version":3,"sources":["/[project]/snapshot/example/chunked/input/index.js"],"sourcesContent":["import { foo } from 'foo'\n\nfoo(true)\n"],"names":[],"mappings":"AAAA;;;AAEA,uIAAI,IAAI,CAAC"}},
+    {"offset": {"line": 6, "column": 0}, "map": {"version": 3, "names": [], "sources": [], "mappings": "A"}}]
+}

crates/turbopack/tests/snapshot/example/chunked/output/snapshot_example_chunked_input_node_modules_foo_index.js.cf8ba911a655ce0a.map

@@ -0,0 +1,6 @@
+{
+  "version": 3,
+  "sections": [
+    {"offset": {"line": 2, "column": 0}, "map": {"version":3,"sources":["/[project]/snapshot/example/chunked/input/node_modules/foo/index.js"],"sourcesContent":["export function foo(value) {\n  console.assert(value);\n}\n"],"names":[],"mappings":"AAAA;;;AAAO,SAAS,GAAG,CAAC,KAAK,EAAE;IACzB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACxB,CAAC"}},
+    {"offset": {"line": 8, "column": 0}, "map": {"version": 3, "names": [], "sources": [], "mappings": "A"}}]
+}