next-dev test runner (vercel#172)

This is a very early version of the next-dev test runner. I'm opening this early to get thoughts from folks re: the direction of the design and implementation.

Fixes vercel#204 

Currently it:
* Discovers integration test fixtures from the filesystem. Right now these are expected to be single files that get bundled and will eventually include assertions. This is powered by the test-generator crate, which allows us not to have to manually enumerate each case. We could consider using this for the node-file-trace tests as well.
* Starts the dev server on a free port and opens a headless browser to its root. The browser control is implemented with the https://crates.io/crates/chromiumoxide crate, which expects Chrome or Chromium to already be available.

Eventually it will:
* [x] Implement a minimal test environment loaded in the browser so that assertions can be run there from bundled code.
* [x] Report back the results of these assertions to rust, where we can pass/fail cargo tests with those results.

In the future it could:
* Possibly include snapshot-style tests to assert on transformed results. This could be in the form of fixture directories instead of files cc @jridgewell
* Support expressing special configuration of turbopack in a fixture, possibly as another file in the fixture directory.
* [x] ~Possibly support distributing tests to a pool of open browsers instead of opening and closing for each test.~

Test Plan: See next PRs

crates/next-dev/Cargo.toml

@@ -24,6 +24,7 @@ tokio_console = [
 anyhow = "1.0.47"
 clap = { version = "3.1.3", features = ["derive"] }
 console-subscriber = { version = "0.1.6", optional = true }
+futures = "0.3.21"
 json = "0.12.4"
 mime = "0.3.16"
 serde = "1.0.136"
@@ -37,5 +38,13 @@ turbopack-core = { path = "../turbopack-core" }
 turbopack-dev-server = { path = "../turbopack-dev-server" }
 webbrowser = "0.7.1"
 
+[dev-dependencies]
+chromiumoxide = { version = "0.3.5", features = [
+  "tokio-runtime",
+], default-features = false }
+lazy_static = "1.4.0"
+portpicker = "0.1.1"
+test-generator = "0.3.0"
+
 [build-dependencies]
 turbo-tasks-build = { path = "../turbo-tasks-build" }

crates/next-dev/src/lib.rs

@@ -23,7 +23,7 @@ mod web_entry_source;
 pub struct NextDevServerBuilder {
     turbo_tasks: Option<Arc<TurboTasks<MemoryBackend>>>,
     project_dir: Option<String>,
-    entry_asset_path: Option<String>,
+    entry_assets: Vec<String>,
     eager_compile: bool,
     hostname: Option<IpAddr>,
     port: Option<u16>,
@@ -40,7 +40,7 @@ impl NextDevServerBuilder {
         NextDevServerBuilder {
             turbo_tasks: None,
             project_dir: None,
-            entry_asset_path: None,
+            entry_assets: vec![],
             eager_compile: false,
             hostname: None,
             port: None,
@@ -57,8 +57,8 @@ impl NextDevServerBuilder {
         self
     }
 
-    pub fn entry_asset_path(mut self, entry_asset_path: String) -> NextDevServerBuilder {
-        self.entry_asset_path = Some(entry_asset_path);
+    pub fn entry_asset(mut self, entry_asset_path: String) -> NextDevServerBuilder {
+        self.entry_assets.push(entry_asset_path);
         self
     }
 
@@ -109,12 +109,10 @@ impl NextDevServerBuilder {
                 let dev_server_fs = DevServerFileSystemVc::new().as_file_system();
                 let main_source = create_web_entry_source(
                     FileSystemPathVc::new(fs, ""),
-                    FileSystemPathVc::new(
-                        fs,
-                        &self
-                            .entry_asset_path
-                            .context("entry_asset_path must be set")?,
-                    ),
+                    self.entry_assets
+                        .iter()
+                        .map(|a| FileSystemPathVc::new(fs, a))
+                        .collect(),
                     dev_server_fs,
                     self.eager_compile,
                 );

crates/next-dev/src/main.rs

@@ -68,7 +68,7 @@ async fn main() {
     let server = NextDevServerBuilder::new()
         .turbo_tasks(tt)
         .project_dir(dir)
-        .entry_asset_path("src/index.js".into())
+        .entry_asset("src/index.js".into())
         .eager_compile(args.eager_compile)
         .hostname(args.hostname)
         .port(args.port)

crates/next-dev/src/web_entry_source.rs

@@ -1,4 +1,5 @@
 use anyhow::{anyhow, Result};
+use futures::future::try_join_all;
 use turbo_tasks::Value;
 use turbo_tasks_fs::{FileSystemPathVc, FileSystemVc};
 use turbopack::ModuleAssetContextVc;
@@ -21,11 +22,10 @@ use crate::EcmascriptModuleAssetVc;
 #[turbo_tasks::function]
 pub async fn create_web_entry_source(
     root: FileSystemPathVc,
-    entry_path: FileSystemPathVc,
+    entry_paths: Vec<FileSystemPathVc>,
     dev_server_fs: FileSystemVc,
     eager_compile: bool,
 ) -> Result<ContentSourceVc> {
-    let source_asset = SourceAssetVc::new(entry_path).into();
     let context: AssetContextVc = ModuleAssetContextVc::new(
         root,
         EnvironmentVc::new(
@@ -42,38 +42,38 @@ pub async fn create_web_entry_source(
         ),
     )
     .into();
-    let module = context.process(source_asset);
+
     let chunking_context: DevChunkingContextVc = DevChunkingContext {
         context_path: root,
         chunk_root_path: FileSystemPathVc::new(dev_server_fs, "/_next/chunks"),
         asset_root_path: FileSystemPathVc::new(dev_server_fs, "/_next/static"),
     }
     .into();
-    let entry_asset =
+
+    let modules = entry_paths
+        .into_iter()
+        .map(|p| context.process(SourceAssetVc::new(p).into()));
+    let chunks = try_join_all(modules.map(|module| async move {
         if let Some(ecmascript) = EcmascriptModuleAssetVc::resolve_from(module).await? {
-            let chunk = ecmascript.as_evaluated_chunk(chunking_context.into());
-            let chunk_group = ChunkGroupVc::from_chunk(chunk);
-            DevHtmlAsset {
-                path: FileSystemPathVc::new(dev_server_fs, "index.html"),
-                chunk_group,
-            }
-            .cell()
-            .into()
+            Ok(ecmascript.as_evaluated_chunk(chunking_context.into()))
         } else if let Some(chunkable) = ChunkableAssetVc::resolve_from(module).await? {
-            let chunk = chunkable.as_chunk(chunking_context.into());
-            let chunk_group = ChunkGroupVc::from_chunk(chunk);
-            DevHtmlAsset {
-                path: FileSystemPathVc::new(dev_server_fs, "index.html"),
-                chunk_group,
-            }
-            .cell()
-            .into()
+            Ok(chunkable.as_chunk(chunking_context.into()))
         } else {
             // TODO convert into a serve-able asset
-            return Err(anyhow!(
+            Err(anyhow!(
                 "Entry module is not chunkable, so it can't be used to bootstrap the application"
-            ));
-        };
+            ))
+        }
+        // ChunkGroupVc::from_chunk(m)
+    }))
+    .await?;
+
+    let entry_asset = DevHtmlAsset {
+        path: FileSystemPathVc::new(dev_server_fs, "index.html"),
+        chunk_groups: chunks.into_iter().map(ChunkGroupVc::from_chunk).collect(),
+    }
+    .cell()
+    .into();
 
     let root_path = FileSystemPathVc::new(dev_server_fs, "");
     let graph = if eager_compile {

crates/next-dev/tests/harness.js

@@ -0,0 +1,7 @@
+import * as jest from 'jest-circus-browser/dist/umd/jest-circus.js'
+import expect from 'expect/build-es5/index.js'
+
+globalThis.__jest__ = jest
+globalThis.expect = expect
+globalThis.describe = jest.describe
+globalThis.it = jest.it

crates/next-dev/tests/integration.rs

@@ -0,0 +1,136 @@
+#![cfg(test)]
+extern crate test_generator;
+
+use std::{net::SocketAddr, path::Path};
+
+use chromiumoxide::browser::{Browser, BrowserConfig};
+use futures::StreamExt;
+use next_dev::{register, NextDevServerBuilder};
+use serde::Deserialize;
+use test_generator::test_resources;
+use turbo_tasks::TurboTasks;
+use turbo_tasks_memory::MemoryBackend;
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct JestRunResult {
+    test_results: Vec<JestTestResult>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct JestTestResult {
+    test_path: Vec<String>,
+    errors: Vec<String>,
+}
+
+#[test_resources("crates/next-dev/tests/integration/*/*/*")]
+#[tokio::main]
+async fn test(resource: &str) {
+    register();
+    let path = Path::new(resource)
+        // test_resources matches and returns relative paths from the workspace root,
+        // but pwd in cargo tests is the crate under test.
+        .strip_prefix("crates/next-dev")
+        .unwrap();
+    assert!(path.exists(), "{} does not exist", resource);
+
+    assert!(
+        path.is_dir(),
+        "{} is not a directory. Integration tests must be directories.",
+        path.to_str().unwrap()
+    );
+
+    if path.ends_with("__skipped__") {
+        // "Skip" directories named `__skipped__`, which include test directories to
+        // skip. These tests are not considered truly skipped by `cargo test`, but they
+        // are not run.
+        return;
+    }
+
+    let test_entry = path.join("index.js");
+    assert!(
+        test_entry.exists(),
+        "Test entry {} must exist.",
+        test_entry.to_str().unwrap()
+    );
+
+    let server = NextDevServerBuilder::new()
+        .turbo_tasks(TurboTasks::new(MemoryBackend::new()))
+        .project_dir("tests".into())
+        .entry_asset("harness.js".into())
+        .entry_asset(
+            test_entry
+                .strip_prefix("tests")
+                .unwrap()
+                .to_str()
+                .unwrap()
+                .replace('\\', "/"),
+        )
+        .eager_compile(false)
+        .hostname("127.0.0.1".parse().unwrap())
+        .port(portpicker::pick_unused_port().unwrap())
+        .build()
+        .await
+        .unwrap();
+
+    println!("server started at http://{}", server.addr);
+
+    tokio::select! {
+        r = run_browser_test(server.addr) => r.unwrap(),
+        r = server.future => r.unwrap(),
+    };
+}
+
+async fn create_browser() -> Result<Browser, Box<dyn std::error::Error>> {
+    let (browser, mut handler) = Browser::launch(BrowserConfig::builder().build()?).await?;
+    // See https://crates.io/crates/chromiumoxide
+    tokio::task::spawn(async move {
+        loop {
+            let _ = handler.next().await.unwrap();
+        }
+    });
+
+    Ok(browser)
+}
+
+async fn run_browser_test(addr: SocketAddr) -> Result<(), Box<dyn std::error::Error>> {
+    let browser = create_browser().await?;
+
+    let page = browser.new_page(format!("http://{}", addr)).await?;
+
+    let run_result: JestRunResult = page
+        .evaluate("__jest__.run()")
+        .await
+        .unwrap()
+        .into_value()
+        .unwrap();
+
+    assert!(
+        !run_result.test_results.is_empty(),
+        "Expected one or more tests to run."
+    );
+
+    let mut messages = vec![];
+    for test_result in run_result.test_results {
+        // It's possible to fail multiple tests across these tests,
+        // so collect them and fail the respective test in Rust with
+        // an aggregate message.
+        if !test_result.errors.is_empty() {
+            messages.push(format!(
+                "\"{}\":\n{}",
+                test_result.test_path[1..].join(" > "),
+                test_result.errors.join("\n")
+            ));
+        }
+    }
+
+    if !messages.is_empty() {
+        panic!(
+            "Failed with error(s) in the following test(s):\n\n{}",
+            messages.join("\n\n--\n")
+        )
+    }
+
+    Ok(())
+}

crates/next-dev/tests/integration/turbopack/basic/simple/index.js

@@ -0,0 +1,19 @@
+it('runs sync tests', () => {
+  expect(true).toBe(true)
+})
+
+it('runs async tests', async () => {
+  await Promise.resolve()
+  expect(true).toBe(true)
+})
+
+describe('nested describe', () => {
+  it('runs sync tests', () => {
+    expect(true).toBe(true)
+  })
+
+  it('runs async tests', async () => {
+    await Promise.resolve()
+    expect(true).toBe(true)
+  })
+})

crates/next-dev/tests/package.json

@@ -0,0 +1,10 @@
+{
+  "private": true,
+  "devDependencies": {
+    "expect": "^24.5.0",
+    "jest-circus-browser": "^1.0.7"
+  },
+  "installConfig": {
+    "hoistingLimits": "workspaces"
+  }
+}

crates/turbopack-dev-server/src/html.rs

@@ -9,7 +9,7 @@ use turbopack_core::{
 #[turbo_tasks::value(shared)]
 pub struct DevHtmlAsset {
     pub path: FileSystemPathVc,
-    pub chunk_group: ChunkGroupVc,
+    pub chunk_groups: Vec<ChunkGroupVc>,
 }
 
 #[turbo_tasks::value_impl]
@@ -26,14 +26,17 @@ impl Asset for DevHtmlAsset {
         let mut scripts = Vec::new();
         let mut stylesheets = Vec::new();
 
-        for chunk in self.chunk_group.chunks().await?.iter() {
-            if let Some(p) = context_path.get_relative_path_to(&*chunk.as_asset().path().await?) {
-                if p.ends_with(".js") {
-                    scripts.push(format!("<script src=\"{}\"></script>", p));
-                } else if p.ends_with(".css") {
-                    stylesheets.push(format!("<link rel=\"stylesheet\" href=\"{}\">", p));
-                } else {
-                    return Err(anyhow!("chunk with unknown asset type: {}", p));
+        for chunk_group in &self.chunk_groups {
+            for chunk in chunk_group.chunks().await?.iter() {
+                if let Some(p) = context_path.get_relative_path_to(&*chunk.as_asset().path().await?)
+                {
+                    if p.ends_with(".js") {
+                        scripts.push(format!("<script src=\"{}\"></script>", p));
+                    } else if p.ends_with(".css") {
+                        stylesheets.push(format!("<link rel=\"stylesheet\" href=\"{}\">", p));
+                    } else {
+                        return Err(anyhow!("chunk with unknown asset type: {}", p));
+                    }
                 }
             }
         }
@@ -50,10 +53,12 @@ impl Asset for DevHtmlAsset {
 
     #[turbo_tasks::function]
     async fn references(&self) -> Result<AssetReferencesVc> {
-        let chunks = self.chunk_group.chunks().await?;
         let mut references = Vec::new();
-        for chunk in chunks.iter() {
-            references.push(ChunkReferenceVc::new(*chunk).into());
+        for chunk_group in &self.chunk_groups {
+            let chunks = chunk_group.chunks().await?;
+            for chunk in chunks.iter() {
+                references.push(ChunkReferenceVc::new(*chunk).into());
+            }
         }
         Ok(AssetReferencesVc::cell(references))
     }