Move examples tests to their own workspace (vercel#4630)

This PR: - Makes `examples_tests` an official workspace in `turborepo-tests/examples` - Extracts some helpers from `turborepo-tests/integration` into their own workspace - This will cause examples tests to run when those helpers change, avoiding issues like vercel#4626. - Make `examples/` a workspace, and sets up a package.json dependency, so that when examples change, examples_tests are also invalidated. (Note that this doesn't affect when CI runs since that's configured in Github Actions workflow configs) --------- Co-authored-by: Nathan Hammond <[email protected]>
itaztelikeyou · Apr 19, 2023 · d3d660d · d3d660d
1 parent a24301d
commit d3d660d
Show file tree

Hide file tree

Showing 129 changed files with 251 additions and 156 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -158,8 +158,7 @@ jobs:
         with:
           PATTERNS: |
             examples/**
-            examples_tests/**
-            scripts/run-example.sh
+            turborepo-tests/examples/**
             !**.md
             !**.mdx
 
@@ -269,7 +268,7 @@ jobs:
           key: prysk-venv-${{ matrix.os.name }}
 
       - name: Integration Tests
-        run: pnpm test -- --filter=turborepo-integration-tests
+        run: pnpm test -- --filter=turborepo-tests-integration
         env:
           GO_TAG: rust
 
@@ -396,7 +395,7 @@ jobs:
           TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
           TURBO_TEAM: ${{ secrets.TURBO_TEAM }}
           TURBO_REMOTE_ONLY: true
-        run: pnpm -- turbo run run-example -- "${{ matrix.example }}" "${{ matrix.manager }}"
+        run: pnpm -- turbo run test --filter="turborepo-tests-examples" -- "${{ matrix.example }}" "${{ matrix.manager }}"
 
   rust_prepare:
     name: Check rust crates

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -10,7 +10,6 @@ Thanks for your interest in contributing to Turbo!
     - [TLS Implementation](#tls-implementation)
     - [Running Turborepo Tests](#running-turborepo-tests)
       - [Go Tests](#go-tests)
-      - [Rust Tests](#rust-tests)
   - [Debugging Turborepo](#debugging-turborepo)
   - [Benchmarking Turborepo](#benchmarking-turborepo)
   - [Updating `turbo`](#updating-turbo)
@@ -50,28 +49,44 @@ to the build command. This allows for us to build for more platforms, as `native
 
 ### Running Turborepo Tests
 
-Dependencies
+Install dependencies
 
-1. Install `jq`, `sponge`, and `zstd`
+On macOS:
 
-On macOS: `brew install sponge jq zstd`
+```bash
+brew install moreutils jq zstd # (moreutils is for sponge)
+```
 
 #### Go Tests
 
-From the root directory, you can
-
-- run unit tests with `pnpm run --filter=cli test`
-- run integration tests with `pnpm test -- --filter=turborepo-tests`
-- run e2e tests with `pnpm run --filter=cli e2e`
-
-To run a single Go test, you can run `go test ./[path/to/package/]`. See more [in the Go docs](https://pkg.go.dev/cmd/go#hdr-Test_packages).
-
-#### Rust Tests
-
-The recommended way to run tests is: `cargo nextest run -p turborepo-lib --features rustls-tls`.
-You'll have to [install it first](https://nexte.st/book/pre-built-binaries.html).
-
-You can also use the built in [`cargo test`](https://doc.rust-lang.org/cargo/commands/cargo-test.html) directly `cargo test -p turborepo-lib`.
+From the root directory, you can run:
+
+- Go unit tests
+  ```bash
+  pnpm test -- --filter=cli
+  ```
+- A single Go unit test (see more [in the Go docs](https://pkg.go.dev/cmd/go#hdr-Test_packages))
+  ```bash
+  cd cli && go test ./[path/to/package/]
+  ```
+- Rust unit tests ([install `nextest` first](https://nexte.st/book/pre-built-binaries.html))
+  ```bash
+  cargo nextest run -p turborepo-lib --features rustls-tls
+  ```
+  You can also use the built in [`cargo test`](https://doc.rust-lang.org/cargo/commands/cargo-test.html)
+  directly with `cargo test -p turborepo-lib`.
+- CLI Integration tests
+  ```bash
+  pnpm test -- --filter=turborepo-tests-integration
+  ```
+- E2E test
+  ```bash
+  pnpm -- turbo e2e --filter=cli
+  ```
+- Example tests
+  ```bash
+  pnpm test -- --filter=turborepo-tests-examples -- <example-name> <packagemanager>
+  ```
 
 ## Debugging Turborepo
 

diff --git a/examples/package.json b/examples/package.json
@@ -0,0 +1,3 @@
+{
+  "name": "turborepo-examples"
+}
diff --git a/examples/turbo.json b/examples/turbo.json
@@ -0,0 +1,8 @@
+{
+  "extends": ["//"],
+  "pipeline": {
+    // This needs to exist because upstream packages depend on it and
+    // it's not defined at the root level.
+    "topo": {}
+  }
+}
diff --git a/package.json b/package.json
@@ -15,7 +15,6 @@
     "turbo": "pnpm run build:turbo && node turbow.js",
     "turbo-prebuilt": "node turbow.js",
     "docs": "pnpm -- turbo run dev --filter=docs --no-cache",
-    "run-example": "./scripts/run-example.sh",
     "prepare": "husky install",
     "test": "pnpm -- turbo run test"
   },

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
@@ -4,10 +4,11 @@ packages:
   - "cli"
   - "packages/*"
   - "benchmark"
-  - "turborepo-tests/integration"
+  - "turborepo-tests/*"
   # Intentionally exclude the `npm` `turbo` package from the workspaces.
   - "!packages/turbo"
   - "packages/turbo-tracing-next-plugin/test/with-mongodb-mongoose"
   - "crates/next-dev-tests/test-harness"
   - "crates/next-dev-tests/tests"
   - "crates/*/js"
+  - examples
diff --git a/turbo.json b/turbo.json
@@ -27,19 +27,6 @@
     "build": {
       "outputs": ["dist/**/*", ".next/**/*"],
       "dependsOn": ["^build"]
-    },
-    "//#run-example": {
-      "inputs": [
-        "examples/**/*.ts",
-        "examples/**/*.tsx",
-        "examples/**/*.json",
-        "examples/**/*.js",
-        "examples/**/*.yaml",
-        "cli/**/*.go",
-        "cli/go.mod",
-        "cli/go.sum",
-        "./scripts/run-example.sh"
-      ]
     }
   }
 }
diff --git a/turborepo-tests/examples/README.md b/turborepo-tests/examples/README.md
@@ -0,0 +1,8 @@
+# `turborepo-tests/examples`
+
+The tests in this directory exercise the `examples/` directory. They validate
+that when someone uses those examples (likely via `npx create-turbo -e <example>`),
+the example works.
+
+These tests do _not_ use a local `turbo` build, they use the version installed in each
+of the `examples/*` directories in their respective `package.json`s.
diff --git a/turborepo-tests/examples/package.json b/turborepo-tests/examples/package.json
@@ -0,0 +1,10 @@
+{
+  "name": "turborepo-tests-examples",
+  "scripts": {
+    "test": "./run-example.sh"
+  },
+  "dependencies": {
+    "turborepo-tests-helpers": "workspace:*",
+    "turborepo-examples": "workspace:*"
+  }
+}
diff --git a/scripts/run-example.sh → turborepo-tests/examples/run-example.sh b/scripts/run-example.sh → turborepo-tests/examples/run-example.sh
@@ -8,7 +8,7 @@ python3 -m venv .cram_env
 export folder=$1
 export pkgManager=$2
 
-TEST_FILE="examples_tests/$2-$1.t"
+TEST_FILE="tests/$2-$1.t"
 
 if [ -f "$TEST_FILE" ]; then
   echo "Running $TEST_FILE"

diff --git a/examples_tests/npm-non-monorepo.t → ...o-tests/examples/tests/npm-non-monorepo.t b/examples_tests/npm-non-monorepo.t → ...o-tests/examples/tests/npm-non-monorepo.t
diff --git a/examples_tests/npm-with-npm.t → ...orepo-tests/examples/tests/npm-with-npm.t b/examples_tests/npm-with-npm.t → ...orepo-tests/examples/tests/npm-with-npm.t
diff --git a/examples_tests/npm-with-yarn.t → ...repo-tests/examples/tests/npm-with-yarn.t b/examples_tests/npm-with-yarn.t → ...repo-tests/examples/tests/npm-with-yarn.t
diff --git a/examples_tests/pnpm-basic.t → turborepo-tests/examples/tests/pnpm-basic.t b/examples_tests/pnpm-basic.t → turborepo-tests/examples/tests/pnpm-basic.t
diff --git a/examples_tests/pnpm-gatsby.t → turborepo-tests/examples/tests/pnpm-gatsby.t b/examples_tests/pnpm-gatsby.t → turborepo-tests/examples/tests/pnpm-gatsby.t
diff --git a/examples_tests/pnpm-kitchen-sink.t → ...-tests/examples/tests/pnpm-kitchen-sink.t b/examples_tests/pnpm-kitchen-sink.t → ...-tests/examples/tests/pnpm-kitchen-sink.t
diff --git a/examples_tests/pnpm-with-svelte.t → ...o-tests/examples/tests/pnpm-with-svelte.t b/examples_tests/pnpm-with-svelte.t → ...o-tests/examples/tests/pnpm-with-svelte.t
diff --git a/examples_tests/setup.sh → turborepo-tests/examples/tests/setup.sh b/examples_tests/setup.sh → turborepo-tests/examples/tests/setup.sh
@@ -6,19 +6,20 @@ pkgManager=$2
 
 # Copy the example dir over to the test dir that prysk puts you in
 SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
-EXAMPLE_DIR="../examples/$exampleName"
+MONOREPO_ROOT_DIR="$SCRIPT_DIR/../../.."
+EXAMPLE_DIR="$MONOREPO_ROOT_DIR/examples/$exampleName"
+
 TARGET_DIR="$(pwd)"
-cp -a "${SCRIPT_DIR}/$EXAMPLE_DIR/." "${TARGET_DIR}/"
+
+cp -a "$EXAMPLE_DIR/." "${TARGET_DIR}/"
 
 # cleanup lockfiles so we can install from scratch
 [ ! -f yarn.lock ] || mv yarn.lock yarn.lock.bak
 [ ! -f pnpm-lock.yaml ] || mv pnpm-lock.yaml pnpm-lock.yaml.bak
 [ ! -f package-lock.json ] || mv package-lock.json package-lock.json.bak
 
-# $TESTDIR is set by prysk to be the directory the test script is in
-# (not this setup.sh script, but it happens to be the same.
-SOURCE_TURBO_DIR="$TESTDIR/../cli"
-TURBO_VERSION_FILE="${SOURCE_TURBO_DIR}/../version.txt"
+
+TURBO_VERSION_FILE="${MONOREPO_ROOT_DIR}/version.txt"
 # Change package.json in the example directory to point to @canary if our branch is currently at that version
 TURBO_TAG=$(cat "$TURBO_VERSION_FILE" | sed -n '2 p')
 if [ "$TURBO_TAG" == "canary" ]; then
@@ -46,4 +47,5 @@ fi
 
 # Delete .git directory if it's there, we'll set up a new git repo
 [ ! -d .git ] || rm -rf .git
-"${SCRIPT_DIR}/../turborepo-tests/integration/tests/_helpers/setup_git.sh" "${TARGET_DIR}"
+
+"$MONOREPO_ROOT_DIR/turborepo-tests/helpers/setup_git.sh" "${TARGET_DIR}"
diff --git a/examples_tests/yarn-non-monorepo.t → ...-tests/examples/tests/yarn-non-monorepo.t b/examples_tests/yarn-non-monorepo.t → ...-tests/examples/tests/yarn-non-monorepo.t
diff --git a/examples_tests/yarn-with-npm.t → ...repo-tests/examples/tests/yarn-with-npm.t b/examples_tests/yarn-with-npm.t → ...repo-tests/examples/tests/yarn-with-npm.t
diff --git a/examples_tests/yarn-with-yarn.t → ...epo-tests/examples/tests/yarn-with-yarn.t b/examples_tests/yarn-with-yarn.t → ...epo-tests/examples/tests/yarn-with-yarn.t
diff --git a/turborepo-tests/examples/turbo.json b/turborepo-tests/examples/turbo.json
@@ -0,0 +1,14 @@
+{
+  "extends": ["//"],
+  "pipeline": {
+    // This is a synthetic tasks that lets us pull in other workspaces as dependencies
+    // So changes in internal workspaces that we depend on, will trigger this task.
+    "topo": {
+      "dependsOn": ["^topo"]
+    },
+    "test": {
+      "dependsOn": ["cli#build", "topo"],
+      "outputs": []
+    }
+  }
+}
diff --git a/turborepo-tests/helpers/package.json b/turborepo-tests/helpers/package.json
@@ -0,0 +1,3 @@
+{
+  "name": "turborepo-tests-helpers"
+}
diff --git a/...tests/integration/tests/_helpers/setup.sh → turborepo-tests/helpers/setup.sh b/...tests/integration/tests/_helpers/setup.sh → turborepo-tests/helpers/setup.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
-ROOT_DIR="${TESTDIR}/../../../.."
+THIS_DIR=$(dirname "${BASH_SOURCE[0]}")
+ROOT_DIR="${THIS_DIR}/../.."
 
 TURBO=${ROOT_DIR}/target/debug/turbo
 VERSION=${ROOT_DIR}/version.txt
diff --git a/...s/integration/tests/_helpers/setup_git.sh → turborepo-tests/helpers/setup_git.sh b/...s/integration/tests/_helpers/setup_git.sh → turborepo-tests/helpers/setup_git.sh
diff --git a/turborepo-tests/helpers/turbo.json b/turborepo-tests/helpers/turbo.json
@@ -0,0 +1,8 @@
+{
+  "extends": ["//"],
+  "pipeline": {
+    // This needs to exist because upstream packages depend on it and
+    // it's not defined at the root level.
+    "topo": {}
+  }
+}
diff --git a/turborepo-tests/integration/package.json b/turborepo-tests/integration/package.json
@@ -1,5 +1,5 @@
 {
-  "name": "turborepo-integration-tests",
+  "name": "turborepo-tests-integration",
   "scripts": {
     "test:setup_python": "python3 -m venv .cram_env",
     "test:setup_pip": "pnpm test:setup_python && .cram_env/bin/python3 -m pip install --quiet --upgrade pip",
@@ -8,5 +8,8 @@
     "test:interactive": ".cram_env/bin/prysk -i --shell=`which bash` tests/**/*.t",
     "test:parallel": ".cram_env/bin/pytest -n auto tests/**/*.t --prysk-shell=`which bash`",
     "pretest:parallel": ".cram_env/bin/pip3 install --quiet pytest \"prysk[pytest-plugin]\" pytest-xdist"
+  },
+  "dependencies": {
+    "turborepo-tests-helpers": "workspace:*"
   }
 }
diff --git a/turborepo-tests/integration/tests/_helpers/setup_monorepo.sh b/turborepo-tests/integration/tests/_helpers/setup_monorepo.sh
@@ -1,10 +1,13 @@
 #!/bin/bash
 
 SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
-TARGET_DIR=$1
 FIXTURE="_fixtures/${2-basic_monorepo}"
+TURBOREPO_TESTS_DIR="$SCRIPT_DIR/../../.."
+
+TARGET_DIR=$1
 cp -a ${SCRIPT_DIR}/../$FIXTURE/. ${TARGET_DIR}/
-${SCRIPT_DIR}/setup_git.sh ${TARGET_DIR}
+
+${TURBOREPO_TESTS_DIR}/helpers/setup_git.sh ${TARGET_DIR}
 
 # Update package manager
 if [ "$3" != "" ]; then

diff --git a/turborepo-tests/integration/tests/absolute_path_warning.t b/turborepo-tests/integration/tests/absolute_path_warning.t
@@ -1,5 +1,5 @@
 Setup
-  $ . ${TESTDIR}/_helpers/setup.sh
+  $ . ${TESTDIR}/../../helpers/setup.sh
   $ . ${TESTDIR}/_helpers/setup_monorepo.sh $(pwd) invalid_turbo_json
 
 Expect warnings

diff --git a/turborepo-tests/integration/tests/bad_flag.t b/turborepo-tests/integration/tests/bad_flag.t
@@ -1,5 +1,5 @@
 Setup
-  $ . ${TESTDIR}/_helpers/setup.sh
+  $ . ${TESTDIR}/../../helpers/setup.sh
 
 Bad flag should print misuse text
   $ ${TURBO} --bad-flag

diff --git a/turborepo-tests/integration/tests/cache_state.t b/turborepo-tests/integration/tests/cache_state.t
@@ -1,5 +1,5 @@
 Setup
-  $ . ${TESTDIR}/_helpers/setup.sh
+  $ . ${TESTDIR}/../../helpers/setup.sh
   $ . ${TESTDIR}/_helpers/setup_monorepo.sh $(pwd)
 
 Run a build to get a local cache.

diff --git a/turborepo-tests/integration/tests/composable_config/composing-add-keys.t b/turborepo-tests/integration/tests/composable_config/composing-add-keys.t
@@ -1,5 +1,5 @@
 Setup
-  $ . ${TESTDIR}/../_helpers/setup.sh
+  $ . ${TESTDIR}/../../../helpers/setup.sh
   $ . ${TESTDIR}/../_helpers/setup_monorepo.sh $(pwd) composable_config
 
 # The add-keys-task in the root turbo.json has no config. This test:

diff --git a/turborepo-tests/integration/tests/composable_config/composing-add-tasks.t b/turborepo-tests/integration/tests/composable_config/composing-add-tasks.t
@@ -1,5 +1,5 @@
 Setup
-  $ . ${TESTDIR}/../_helpers/setup.sh
+  $ . ${TESTDIR}/../../../helpers/setup.sh
   $ . ${TESTDIR}/../_helpers/setup_monorepo.sh $(pwd) composable_config
 
   $ ${TURBO} run added-task --filter=add-tasks

diff --git a/turborepo-tests/integration/tests/composable_config/composing-bad-json.t b/turborepo-tests/integration/tests/composable_config/composing-bad-json.t
@@ -1,5 +1,5 @@
 Setup
-  $ . ${TESTDIR}/../_helpers/setup.sh
+  $ . ${TESTDIR}/../../../helpers/setup.sh
   $ . ${TESTDIR}/../_helpers/setup_monorepo.sh $(pwd) composable_config
 
 # Put some bad JSON into the turbo.json in this app

diff --git a/turborepo-tests/integration/tests/composable_config/composing-cache.t b/turborepo-tests/integration/tests/composable_config/composing-cache.t
@@ -1,5 +1,5 @@
 Setup
-  $ . ${TESTDIR}/../_helpers/setup.sh
+  $ . ${TESTDIR}/../../../helpers/setup.sh
   $ . ${TESTDIR}/../_helpers/setup_monorepo.sh $(pwd) composable_config
 
 This test covers:

diff --git a/turborepo-tests/integration/tests/composable_config/composing-config-change.t b/turborepo-tests/integration/tests/composable_config/composing-config-change.t
@@ -1,5 +1,5 @@
 Setup
-  $ . ${TESTDIR}/../_helpers/setup.sh
+  $ . ${TESTDIR}/../../../helpers/setup.sh
   $ . ${TESTDIR}/../_helpers/setup_monorepo.sh $(pwd) composable_config
 
 # 1. First run, check the hash

diff --git a/turborepo-tests/integration/tests/composable_config/composing-cross-workspace.t b/turborepo-tests/integration/tests/composable_config/composing-cross-workspace.t
@@ -1,5 +1,5 @@
 Setup
-  $ . ${TESTDIR}/../_helpers/setup.sh
+  $ . ${TESTDIR}/../../../helpers/setup.sh
   $ . ${TESTDIR}/../_helpers/setup_monorepo.sh $(pwd) composable_config
   $ ${TURBO} run cross-workspace-task --filter=cross-workspace
   \xe2\x80\xa2 Packages in scope: cross-workspace (esc)

diff --git a/turborepo-tests/integration/tests/composable_config/composing-invalid-config.t b/turborepo-tests/integration/tests/composable_config/composing-invalid-config.t
@@ -1,5 +1,5 @@
 Setup
-  $ . ${TESTDIR}/../_helpers/setup.sh
+  $ . ${TESTDIR}/../../../helpers/setup.sh
   $ . ${TESTDIR}/../_helpers/setup_monorepo.sh $(pwd) composable_config
 
 # The test is greping from a logfile because the list of errors can appear in any order

diff --git a/...orepo-tests/integration/tests/composable_config/composing-missing-workspace-config-deps.t b/...orepo-tests/integration/tests/composable_config/composing-missing-workspace-config-deps.t
@@ -1,5 +1,5 @@
 Setup
-  $ . ${TESTDIR}/../_helpers/setup.sh
+  $ . ${TESTDIR}/../../../helpers/setup.sh
   $ . ${TESTDIR}/../_helpers/setup_monorepo.sh $(pwd) composable_config
 
 # The missing-workspace-config-task-with-deps configures dependsOn in the root turbo.json.