Implement Ropes for shared string construction (vercel#2525)

lucas8 · Nov 4, 2022 · 2ff414b · 2ff414b
1 parent 5aa3e15
commit 2ff414b
Show file tree

Hide file tree

Showing 35 changed files with 898 additions and 264 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/cli/libturbo.h b/cli/libturbo.h
@@ -1,6 +1,6 @@
 /* Code generated by cmd/cgo; DO NOT EDIT. */
 
-/* package github.com/vercel/turborepo/cli/cmd/turbo */
+/* package github.com/vercel/turbo/cli/cmd/turbo */
 
 
 #line 1 "cgo-builtin-export-prolog"

diff --git a/crates/next-core/src/app_source.rs b/crates/next-core/src/app_source.rs
@@ -1,4 +1,4 @@
-use std::{collections::HashMap, fmt::Write};
+use std::{collections::HashMap, io::Write};
 
 use anyhow::{anyhow, Context, Result};
 use turbo_tasks::{
@@ -7,8 +7,8 @@ use turbo_tasks::{
 };
 use turbo_tasks_env::ProcessEnvVc;
 use turbo_tasks_fs::{
-    DirectoryContent, DirectoryEntry, File, FileContent, FileContentVc, FileSystemEntryType,
-    FileSystemPathVc,
+    rope::RopeBuilder, DirectoryContent, DirectoryEntry, File, FileContent, FileContentVc,
+    FileSystemEntryType, FileSystemPathVc,
 };
 use turbopack::{
     ecmascript::EcmascriptInputTransform,
@@ -494,15 +494,17 @@ impl NodeEntry for AppRenderer {
             .into_iter()
             .try_join()
             .await?;
-        let mut result =
-            "import IPC, { Ipc } from \"@vercel/turbopack-next/internal/ipc\";\n".to_string();
+        let mut result = RopeBuilder::from(
+            "import IPC, { Ipc } from \"@vercel/turbopack-next/internal/ipc\";\n",
+        );
+
         for (_, import) in segments.iter() {
             if let Some((p, identifier, chunks_identifier)) = import {
+                result += r#"("TURBOPACK {{ transition: next-layout-entry; chunking-type: parallel }}");
+"#;
                 writeln!(
                     result,
-                    r#"("TURBOPACK {{ transition: next-layout-entry; chunking-type: parallel }}");
-import {}, {{ chunks as {} }} from {};
-"#,
+                    "import {}, {{ chunks as {} }} from {};\n",
                     identifier,
                     chunks_identifier,
                     stringify_str(p)
@@ -518,7 +520,8 @@ import BOOTSTRAP from {};
                 stringify_str(&page)
             )?;
         }
-        result.push_str("const LAYOUT_INFO = [");
+
+        result += "const LAYOUT_INFO = [";
         for (segment_str_lit, import) in segments.iter() {
             if let Some((_, identifier, chunks_identifier)) = import {
                 writeln!(
@@ -530,13 +533,15 @@ import BOOTSTRAP from {};
                 writeln!(result, "  {{ segment: {segment_str_lit} }},",)?
             }
         }
-        result.push_str("];\n\n");
+        result += "];\n\n";
+
         let base_code = next_js_file("entry/app-renderer.tsx");
-        let mut file = File::from(result);
         if let FileContent::Content(base_file) = &*base_code.await? {
-            file.push_content(base_file.content());
+            result += base_file.content()
         }
-        let asset = VirtualAssetVc::new(path.join("entry"), FileContent::Content(file).into());
+
+        let file = File::from(result.build());
+        let asset = VirtualAssetVc::new(path.join("entry"), file.into());
         let (context, intermediate_output_path) = if is_rsc {
             (self.context, self.intermediate_output_path.join("__rsc__"))
         } else {

diff --git a/crates/next-core/src/next_client_component/with_chunks.rs b/crates/next-core/src/next_client_component/with_chunks.rs
@@ -129,7 +129,8 @@ const chunks = {};
 ",
                 module_id,
                 Value::Array(client_chunks)
-            ),
+            )
+            .into(),
             ..Default::default()
         }
         .cell())

diff --git a/crates/next-core/src/next_client_component/with_client_chunks.rs b/crates/next-core/src/next_client_component/with_client_chunks.rs
@@ -136,7 +136,8 @@ const chunks = {};
 ",
                 module_id,
                 Value::Array(client_chunks)
-            ),
+            )
+            .into(),
             ..Default::default()
         }
         .cell())

diff --git a/crates/next-core/src/nodejs/mod.rs b/crates/next-core/src/nodejs/mod.rs
@@ -569,7 +569,7 @@ async fn run_proxy_operation(
     Ok(ProxyResult {
         status,
         headers,
-        body,
+        body: body.into(),
     })
 }
 
@@ -610,7 +610,7 @@ async fn proxy_error(
             "content-type".to_string(),
             "text/html; charset=utf-8".to_string(),
         ],
-        body: body.into_bytes(),
+        body: body.into(),
     }
     .cell())
 }
diff --git a/crates/turbo-tasks-env/src/dotenv.rs b/crates/turbo-tasks-env/src/dotenv.rs
@@ -53,7 +53,7 @@ impl ProcessEnv for DotenvProcessEnv {
                 // from_read will load parse and evalute the Read, and set variables
                 // into the global env. If a later dotenv defines an already defined
                 // var, it'll be ignored.
-                res = dotenvy::from_read(f.content());
+                res = dotenvy::from_read(f.read());
 
                 vars = env::vars().collect();
                 restore_env(&vars, &initial);

diff --git a/crates/turbo-tasks-fs/Cargo.toml b/crates/turbo-tasks-fs/Cargo.toml
@@ -11,6 +11,7 @@ bench = false
 [dependencies]
 anyhow = "1.0.47"
 bitflags = "1.3.2"
+bytes = "1.1.0"
 concurrent-queue = "1.2.2"
 futures = "0.3.24"
 futures-retry = "0.6.0"

diff --git a/crates/turbo-tasks-fs/examples/hash_directory.rs b/crates/turbo-tasks-fs/examples/hash_directory.rs
@@ -5,6 +5,7 @@
 use std::{
     collections::BTreeMap,
     env::current_dir,
+    io::Read,
     time::{Duration, Instant},
 };
 
@@ -86,7 +87,13 @@ async fn hash_directory(directory: FileSystemPathVc) -> Result<StringVc> {
             println!("{}: not found", directory.await?.path);
         }
     };
-    let hash = hash_content(hashes.into_values().collect::<Vec<String>>().join(","));
+    let hash = hash_content(
+        &mut hashes
+            .into_values()
+            .collect::<Vec<String>>()
+            .join(",")
+            .as_bytes(),
+    );
     println!("hash_directory({})", dir_path);
     Ok(hash)
 }
@@ -95,17 +102,20 @@ async fn hash_directory(directory: FileSystemPathVc) -> Result<StringVc> {
 async fn hash_file(file_path: FileSystemPathVc) -> Result<StringVc> {
     let content = file_path.read().await?;
     Ok(match &*content {
-        FileContent::Content(file) => hash_content(file),
+        FileContent::Content(file) => hash_content(&mut file.read()),
         FileContent::NotFound => {
             // report error
             StringVc::cell("".to_string())
         }
     })
 }
 
-fn hash_content(content: impl AsRef<[u8]>) -> StringVc {
+fn hash_content<R: Read>(content: &mut R) -> StringVc {
     let mut hasher = Sha256::new();
-    hasher.update(content);
+    let mut buf = [0; 1024];
+    while let Ok(size) = content.read(&mut buf) {
+        hasher.update(&buf[0..size]);
+    }
     let result = format!("{:x}", hasher.finalize());
 
     StringVc::cell(result)

diff --git a/crates/turbo-tasks-fs/examples/hash_glob.rs b/crates/turbo-tasks-fs/examples/hash_glob.rs
@@ -5,6 +5,7 @@
 use std::{
     collections::BTreeMap,
     env::current_dir,
+    io::Read,
     time::{Duration, Instant},
 };
 
@@ -79,25 +80,34 @@ async fn hash_glob_result(result: ReadGlobResultVc) -> Result<StringVc> {
     if hashes.is_empty() {
         return Ok(empty_string());
     }
-    let hash = hash_content(hashes.into_values().collect::<Vec<String>>().join(","));
+    let hash = hash_content(
+        &mut hashes
+            .into_values()
+            .collect::<Vec<String>>()
+            .join(",")
+            .as_bytes(),
+    );
     Ok(hash)
 }
 
 #[turbo_tasks::function]
 async fn hash_file(file_path: FileSystemPathVc) -> Result<StringVc> {
     let content = file_path.read().await?;
     Ok(match &*content {
-        FileContent::Content(file) => hash_content(file),
+        FileContent::Content(file) => hash_content(&mut file.read()),
         FileContent::NotFound => {
             // report error
             StringVc::cell("".to_string())
         }
     })
 }
 
-fn hash_content(content: impl AsRef<[u8]>) -> StringVc {
+fn hash_content<R: Read>(content: &mut R) -> StringVc {
     let mut hasher = Sha256::new();
-    hasher.update(content);
+    let mut buf = [0; 1024];
+    while let Ok(size) = content.read(&mut buf) {
+        hasher.update(&buf[0..size]);
+    }
     let result = format!("{:x}", hasher.finalize());
 
     StringVc::cell(result)