Experimental Feature: Strict Environments (vercel#4449)

To provide a way to reduce information leakage into the task execution
environment, add a strict mode for environment variable handling. This
mode prevents passing non-enumerated variables to the subprocess of
the task being executed.

Co-authored-by: Mehul Kar <[email protected]>

cli/integration_tests/bad_flag.t

@@ -19,7 +19,7 @@ Bad flag with an implied run command should display run flags
 
     note: to pass '--bad-flag' as a value, use '-- --bad-flag'
 
-  Usage: turbo <--cache-dir <CACHE_DIR>|--cache-workers <CACHE_WORKERS>|--concurrency <CONCURRENCY>|--continue|--dry-run [<DRY_RUN>]|--single-package|--filter <FILTER>|--force|--global-deps <GLOBAL_DEPS>|--graph [<GRAPH>]|--ignore <IGNORE>|--include-dependencies|--no-cache|--no-daemon|--no-deps|--output-logs <OUTPUT_LOGS>|--only|--parallel|--pkg-inference-root <PKG_INFERENCE_ROOT>|--profile <PROFILE>|--remote-only|--scope <SCOPE>|--since <SINCE>|--summarize [<SUMMARIZE>]|--log-prefix <LOG_PREFIX>|TASKS|PASS_THROUGH_ARGS|--experimental-space-id <EXPERIMENTAL_SPACE_ID>>
+  Usage: turbo <--cache-dir <CACHE_DIR>|--cache-workers <CACHE_WORKERS>|--concurrency <CONCURRENCY>|--continue|--dry-run [<DRY_RUN>]|--single-package|--filter <FILTER>|--force|--global-deps <GLOBAL_DEPS>|--graph [<GRAPH>]|--experimental-env-mode [<ENV_MODE>]|--ignore <IGNORE>|--include-dependencies|--no-cache|--no-daemon|--no-deps|--output-logs <OUTPUT_LOGS>|--only|--parallel|--pkg-inference-root <PKG_INFERENCE_ROOT>|--profile <PROFILE>|--remote-only|--scope <SCOPE>|--since <SINCE>|--summarize [<SUMMARIZE>]|--log-prefix <LOG_PREFIX>|TASKS|PASS_THROUGH_ARGS|--experimental-space-id <EXPERIMENTAL_SPACE_ID>>
 
   For more information, try '--help'.
 

cli/integration_tests/strict_env_vars/fixture-configs/all.json

@@ -0,0 +1,12 @@
+{
+  "$schema": "https://turbo.build/schema.json",
+  "pipeline": {
+    "build": {
+      "outputs": ["dist/**"],
+      "experimentalPassThroughEnv": ["LOCAL_VAR_PT"],
+      "env": ["LOCAL_VAR_DEP"]
+    }
+  },
+  "experimentalGlobalPassThroughEnv": ["GLOBAL_VAR_PT"],
+  "globalEnv": ["GLOBAL_VAR_DEP"]
+}

cli/integration_tests/strict_env_vars/fixture-configs/global_pt-empty.json

@@ -0,0 +1,9 @@
+{
+  "$schema": "https://turbo.build/schema.json",
+  "pipeline": {
+    "build": {
+      "outputs": ["dist/**"]
+    }
+  },
+  "experimentalGlobalPassThroughEnv": []
+}

cli/integration_tests/strict_env_vars/fixture-configs/global_pt.json

@@ -0,0 +1,9 @@
+{
+  "$schema": "https://turbo.build/schema.json",
+  "pipeline": {
+    "build": {
+      "outputs": ["dist/**"]
+    }
+  },
+  "experimentalGlobalPassThroughEnv": ["GLOBAL_VAR_PT"]
+}

cli/integration_tests/strict_env_vars/fixture-configs/task_pt-empty.json

@@ -0,0 +1,9 @@
+{
+  "$schema": "https://turbo.build/schema.json",
+  "pipeline": {
+    "build": {
+      "outputs": ["dist/**"],
+      "experimentalPassthroughEnv": []
+    }
+  }
+}

cli/integration_tests/strict_env_vars/fixture-configs/task_pt.json

@@ -0,0 +1,9 @@
+{
+  "$schema": "https://turbo.build/schema.json",
+  "pipeline": {
+    "build": {
+      "outputs": ["dist/**"],
+      "experimentalPassthroughEnv": ["LOCAL_VAR_PT"]
+    }
+  }
+}

cli/integration_tests/strict_env_vars/get-global-hash.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# This script greps stdin (i.e. what's piped to it)
+# splits it by "=" and prints the second value.
+# it's intendted to get the global hash from a debug log that looks like this:
+# 2023-04-06T04:28:19.599Z [DEBUG] turbo: global hash: value=a027dadc4dea675e
+#
+# Usage:
+# turbo build -vv 2>&1 | "$TESTDIR/./get-global-hash.sh"
+#
+#
+grep "global hash:" - | awk '{split($0,a,"="); print a[2]}'

cli/integration_tests/strict_env_vars/global_hash_infer.t

@@ -0,0 +1,22 @@
+Setup
+  $ . ${TESTDIR}/../setup.sh
+  $ . ${TESTDIR}/setup.sh $(pwd) monorepo
+
+With --experimental-env-mode=infer
+
+Baseline global hash
+  $ BASELINE=$(${TURBO} build -vv 2>&1 | "$TESTDIR/./get-global-hash.sh")
+
+There's no config to start, so the global hash does not change when flag is passed
+  $ WITH_FLAG=$(${TURBO} build -vv --experimental-env-mode=infer 2>&1 | "$TESTDIR/./get-global-hash.sh")
+  $ test $BASELINE = $WITH_FLAG
+
+Add empty config for global pass through env var, global hash changes
+  $ cp "$TESTDIR/fixture-configs/global_pt-empty.json" "$(pwd)/turbo.json" && git commit -am "no comment" --quiet
+  $ WITH_EMPTY_GLOBAL=$(${TURBO} build -vv --experimental-env-mode=infer 2>&1 | "$TESTDIR/./get-global-hash.sh")
+  $ test $BASELINE != $WITH_EMPTY_GLOBAL
+
+Add global pass through env var, global hash changes again, because we changed the value
+  $ cp "$TESTDIR/fixture-configs/global_pt.json" "$(pwd)/turbo.json" && git commit -am "no comment" --quiet
+  $ WITH_GLOBAL=$(${TURBO} build -vv --experimental-env-mode=infer 2>&1 | "$TESTDIR/./get-global-hash.sh")
+  $ test $WITH_EMPTY_GLOBAL != $WITH_GLOBAL

cli/integration_tests/strict_env_vars/global_hash_loose.t

@@ -0,0 +1,25 @@
+Setup
+  $ . ${TESTDIR}/../setup.sh
+  $ . ${TESTDIR}/setup.sh $(pwd) monorepo
+
+With --experimental-env-mode=loose
+
+Baseline global hash
+  $ BASELINE=$(${TURBO} build -vv 2>&1 | "$TESTDIR/./get-global-hash.sh")
+
+Hash changes, because we're using a new mode
+  $ WITH_FLAG=$(${TURBO} build -vv --experimental-env-mode=loose 2>&1 | "$TESTDIR/./get-global-hash.sh")
+  $ test $BASELINE != $WITH_FLAG
+
+Add empty config for global pass through env var
+Hash does not change, because in loose mode, we don't care what the actual config contains
+  $ cp "$TESTDIR/fixture-configs/global_pt-empty.json" "$(pwd)/turbo.json" && git commit -am "no comment" --quiet
+  $ WITH_EMPTY_GLOBAL=$(${TURBO} build -vv --experimental-env-mode=loose 2>&1 | "$TESTDIR/./get-global-hash.sh")
+  $ test $WITH_FLAG = $WITH_EMPTY_GLOBAL
+
+Add global pass through env var
+Hash does not change, because in loose mode, we don't care what the actual config contains
+  $ cp "$TESTDIR/fixture-configs/global_pt.json" "$(pwd)/turbo.json" && git commit -am "no comment" --quiet
+  $ WITH_GLOBAL=$(${TURBO} build -vv --experimental-env-mode=loose 2>&1 | "$TESTDIR/./get-global-hash.sh")
+  $ test $WITH_FLAG = $WITH_GLOBAL
+  $ test $WITH_EMPTY_GLOBAL = $WITH_GLOBAL

cli/integration_tests/strict_env_vars/global_hash_no-value.t

@@ -0,0 +1,22 @@
+Setup
+  $ . ${TESTDIR}/../setup.sh
+  $ . ${TESTDIR}/setup.sh $(pwd) monorepo
+
+With --experimental-env-mode (should be the same as --experimental-env-mode=infer)
+
+Baseline global hash
+  $ BASELINE=$(${TURBO} build -vv 2>&1 | "$TESTDIR/./get-global-hash.sh")
+
+There's no config to start, so the global hash does not change when flag is passed
+  $ WITH_FLAG=$(${TURBO} build -vv --experimental-env-mode 2>&1 | "$TESTDIR/./get-global-hash.sh")
+  $ test $BASELINE = $WITH_FLAG
+
+Add empty config for global pass through env var, global hash changes
+  $ cp "$TESTDIR/fixture-configs/global_pt-empty.json" "$(pwd)/turbo.json" && git commit -am "no comment" --quiet
+  $ WITH_EMPTY_GLOBAL=$(${TURBO} build -vv --experimental-env-mode 2>&1 | "$TESTDIR/./get-global-hash.sh")
+  $ test $BASELINE != $WITH_EMPTY_GLOBAL
+
+Add global pass through env var, global hash changes again, because we changed the value
+  $ cp "$TESTDIR/fixture-configs/global_pt.json" "$(pwd)/turbo.json" && git commit -am "no comment" --quiet
+  $ WITH_GLOBAL=$(${TURBO} build -vv --experimental-env-mode 2>&1 | "$TESTDIR/./get-global-hash.sh")
+  $ test $WITH_EMPTY_GLOBAL != $WITH_GLOBAL

cli/integration_tests/strict_env_vars/global_hash_strict.t

@@ -0,0 +1,24 @@
+Setup
+  $ . ${TESTDIR}/../setup.sh
+  $ . ${TESTDIR}/setup.sh $(pwd) monorepo
+
+With strict mode
+
+Get Baseline global hash
+  $ BASELINE=$(${TURBO} build -vv 2>&1 | "$TESTDIR/./get-global-hash.sh")
+
+Hash changes, because we're using a new mode
+  $ WITH_FLAG=$(${TURBO} build -vv --experimental-env-mode=strict 2>&1 | "$TESTDIR/./get-global-hash.sh")
+  $ test $BASELINE != $WITH_FLAG
+
+Add empty config for global pass through env var
+Hash does not change, because the mode is the same and we haven't added any new pass through vars
+  $ cp "$TESTDIR/fixture-configs/global_pt-empty.json" "$(pwd)/turbo.json" && git commit -am "no comment" --quiet
+  $ WITH_EMPTY_GLOBAL=$(${TURBO} build -vv --experimental-env-mode=strict 2>&1 | "$TESTDIR/./get-global-hash.sh")
+  $ test $WITH_FLAG = $WITH_EMPTY_GLOBAL
+
+Add global pass through env var
+Hash changes, because we have a new pass through value
+  $ cp "$TESTDIR/fixture-configs/global_pt.json" "$(pwd)/turbo.json" && git commit -am "no comment" --quiet
+  $ WITH_GLOBAL=$(${TURBO} build -vv --experimental-env-mode=strict 2>&1 | "$TESTDIR/./get-global-hash.sh")
+  $ test $WITH_EMPTY_GLOBAL != $WITH_GLOBAL

cli/integration_tests/strict_env_vars/monorepo/.gitignore

@@ -0,0 +1,4 @@
+node_modules/
+.turbo
+.npmrc
+out.txt

cli/integration_tests/strict_env_vars/monorepo/apps/my-app/build.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+pathset="no"
+sysrootset="no"
+
+if [ ! -z "$PATH" ]; then
+  pathset="yes"
+fi
+
+if [ ! -z "$SYSTEMROOT" ]; then
+  sysrootset="yes"
+fi
+
+{
+  echo -n "globalpt: '$GLOBAL_VAR_PT', "
+  echo -n "localpt: '$LOCAL_VAR_PT', "
+  echo -n "globaldep: '$GLOBAL_VAR_DEP', "
+  echo -n "localdep: '$LOCAL_VAR_DEP', "
+  echo -n "other: '$OTHER_VAR', "
+  echo -n "sysroot set: '$sysrootset', "
+  echo "path set: '$pathset'"
+} > out.txt

cli/integration_tests/strict_env_vars/monorepo/apps/my-app/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "my-app",
+  "scripts": {
+    "build": "./build.sh"
+  }
+}

cli/integration_tests/strict_env_vars/monorepo/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "monorepo",
+  "workspaces": [
+    "apps/**"
+  ]
+}

cli/integration_tests/strict_env_vars/monorepo/turbo.json

@@ -0,0 +1,8 @@
+{
+  "$schema": "https://turbo.build/schema.json",
+  "pipeline": {
+    "build": {
+      "outputs": ["dist/**"]
+    }
+  }
+}

cli/integration_tests/strict_env_vars/setup.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
+TARGET_DIR=$1
+FIXTURE=$2
+
+cp -a ${SCRIPT_DIR}/$2/. ${TARGET_DIR}/
+${SCRIPT_DIR}/../setup_git.sh ${TARGET_DIR}

cli/integration_tests/strict_env_vars/usage_infer.t

@@ -0,0 +1,42 @@
+Setup
+  $ . ${TESTDIR}/../setup.sh
+  $ . ${TESTDIR}/setup.sh $(pwd) monorepo
+
+With --experimental-env-mode=infer
+
+Set the env vars
+  $ export GLOBAL_VAR_PT=higlobalpt
+  $ export GLOBAL_VAR_DEP=higlobaldep
+  $ export LOCAL_VAR_PT=hilocalpt
+  $ export LOCAL_VAR_DEP=hilocaldep
+  $ export OTHER_VAR=hiother
+
+Conditionally set these vars if they aren't already there for the purpose of the test.
+The test doesn't care about the values, it just checks that the var is available to the task
+so we just have to make sure the parent process has them set. In Github CI, for example SHELL
+isn't already set.
+  $ export SYSTEMROOT="${SYSTEMROOT:=hisysroot}"
+  $ export PATH="${PATH:=hipath}"
+
+Inferred mode as loose because no pass through configs, all vars are available
+  $ ${TURBO} build -vv --experimental-env-mode=infer > /dev/null 2>&1
+  $ cat apps/my-app/out.txt
+  globalpt: 'higlobalpt', localpt: 'hilocalpt', globaldep: 'higlobaldep', localdep: 'hilocaldep', other: 'hiother', sysroot set: 'yes', path set: 'yes'
+
+Inferred mode as strict, because global pass through config, no vars available
+  $ cp "$TESTDIR/fixture-configs/global_pt-empty.json" "$(pwd)/turbo.json" && git commit -am "no comment" --quiet
+  $ ${TURBO} build -vv --experimental-env-mode=infer > /dev/null 2>&1
+  $ cat apps/my-app/out.txt
+  globalpt: '', localpt: '', globaldep: '', localdep: '', other: '', sysroot set: 'yes', path set: 'yes'
+
+Inferred mode as strict, because task pass through config, no vars available
+  $ cp "$TESTDIR/fixture-configs/task_pt-empty.json" "$(pwd)/turbo.json" && git commit -am "no comment" --quiet
+  $ ${TURBO} build -vv --experimental-env-mode=infer > /dev/null 2>&1
+  $ cat apps/my-app/out.txt
+  globalpt: '', localpt: '', globaldep: '', localdep: '', other: '', sysroot set: 'yes', path set: 'yes'
+
+Inferred mode as strict, with declared deps and pass through. all declared available, other is not available
+  $ cp "$TESTDIR/fixture-configs/all.json" "$(pwd)/turbo.json" && git commit -am "no comment" --quiet
+  $ ${TURBO} build -vv --experimental-env-mode=infer > /dev/null 2>&1
+  $ cat apps/my-app/out.txt
+  globalpt: 'higlobalpt', localpt: 'hilocalpt', globaldep: 'higlobaldep', localdep: 'hilocaldep', other: '', sysroot set: 'yes', path set: 'yes'

cli/integration_tests/strict_env_vars/usage_loose.t

@@ -0,0 +1,24 @@
+Setup
+  $ . ${TESTDIR}/../setup.sh
+  $ . ${TESTDIR}/setup.sh $(pwd) monorepo
+
+With --experimental-env-mode=loose, all vars are available
+
+Set the env vars
+  $ export GLOBAL_VAR_PT=higlobalpt
+  $ export GLOBAL_VAR_DEP=higlobaldep
+  $ export LOCAL_VAR_PT=hilocalpt
+  $ export LOCAL_VAR_DEP=hilocaldep
+  $ export OTHER_VAR=hiother
+  $ export SYSTEMROOT=hisysroot
+
+All vars available in loose mode
+  $ ${TURBO} build -vv --experimental-env-mode=loose > /dev/null 2>&1
+  $ cat apps/my-app/out.txt
+  globalpt: 'higlobalpt', localpt: 'hilocalpt', globaldep: 'higlobaldep', localdep: 'hilocaldep', other: 'hiother', sysroot set: 'yes', path set: 'yes'
+
+All vars available in loose mode, even when global and pass through configs defined
+  $ cp "$TESTDIR/fixture-configs/all.json" "$(pwd)/turbo.json" && git commit -am "no comment" --quiet
+  $ ${TURBO} build -vv --experimental-env-mode=loose > /dev/null 2>&1
+  $ cat apps/my-app/out.txt
+  globalpt: 'higlobalpt', localpt: 'hilocalpt', globaldep: 'higlobaldep', localdep: 'hilocaldep', other: 'hiother', sysroot set: 'yes', path set: 'yes'

cli/integration_tests/strict_env_vars/usage_strict.t

@@ -0,0 +1,24 @@
+Setup
+  $ . ${TESTDIR}/../setup.sh
+  $ . ${TESTDIR}/setup.sh $(pwd) monorepo
+
+With --experimental-env-mode=strict, only declared vars are available
+
+Set the env vars
+  $ export GLOBAL_VAR_PT=higlobalpt
+  $ export GLOBAL_VAR_DEP=higlobaldep
+  $ export LOCAL_VAR_PT=hilocalpt
+  $ export LOCAL_VAR_DEP=hilocaldep
+  $ export OTHER_VAR=hiother
+  $ export SYSTEMROOT=hisysroot
+
+No vars available by default
+  $ ${TURBO} build -vv --experimental-env-mode=strict > /dev/null 2>&1
+  $ cat apps/my-app/out.txt
+  globalpt: '', localpt: '', globaldep: '', localdep: '', other: '', sysroot set: 'yes', path set: 'yes'
+
+All declared vars available, others are not available
+  $ cp "$TESTDIR/fixture-configs/all.json" "$(pwd)/turbo.json" && git commit -am "no comment" --quiet
+  $ ${TURBO} build -vv --experimental-env-mode=strict > /dev/null 2>&1
+  $ cat apps/my-app/out.txt
+  globalpt: 'higlobalpt', localpt: 'hilocalpt', globaldep: 'higlobaldep', localdep: 'hilocaldep', other: '', sysroot set: 'yes', path set: 'yes'

cli/internal/core/engine.go

@@ -140,14 +140,14 @@ func (e *Engine) getTaskDefinition(pkg string, taskName string, taskID string) (
 	if task, ok := pipeline[taskID]; ok {
 		return &Task{
 			Name:           taskName,
-			TaskDefinition: task.TaskDefinition,
+			TaskDefinition: task.GetTaskDefinition(),
 		}, nil
 	}
 
 	if task, ok := pipeline[taskName]; ok {
 		return &Task{
 			Name:           taskName,
-			TaskDefinition: task.TaskDefinition,
+			TaskDefinition: task.GetTaskDefinition(),
 		}, nil
 	}
 

cli/internal/env/env.go

@@ -35,6 +35,11 @@ func (evm EnvironmentVariableMap) Merge(another EnvironmentVariableMap) {
 	}
 }
 
+// Add creates one new environment variable.
+func (evm EnvironmentVariableMap) Add(key string, value string) {
+	evm[key] = value
+}
+
 // Names returns a sorted list of env var names for the EnvironmentVariableMap
 func (evm EnvironmentVariableMap) Names() []string {
 	names := []string{}
@@ -85,7 +90,8 @@ func (evm EnvironmentVariableMap) ToHashable() EnvironmentVariablePairs {
 	})
 }
 
-func getEnvMap() EnvironmentVariableMap {
+// GetEnvMap returns a map of env vars and their values from os.Environ
+func GetEnvMap() EnvironmentVariableMap {
 	envMap := make(map[string]string)
 	for _, envVar := range os.Environ() {
 		if i := strings.Index(envVar, "="); i >= 0 {
@@ -96,8 +102,8 @@ func getEnvMap() EnvironmentVariableMap {
 	return envMap
 }
 
-// fromKeys returns a map of env vars and their values from a given set of env var names
-func fromKeys(all EnvironmentVariableMap, keys []string) EnvironmentVariableMap {
+// FromKeys returns a map of env vars and their values from a given set of env var names
+func FromKeys(all EnvironmentVariableMap, keys []string) EnvironmentVariableMap {
 	output := EnvironmentVariableMap{}
 	for _, key := range keys {
 		output[key] = all[key]
@@ -138,14 +144,14 @@ func fromMatching(all EnvironmentVariableMap, keyMatchers []string, shouldExclud
 
 // GetHashableEnvVars returns all sorted key=value env var pairs for both frameworks and from envKeys
 func GetHashableEnvVars(keys []string, matchers []string, envVarContainingExcludePrefix string) (DetailedMap, error) {
-	all := getEnvMap()
+	all := GetEnvMap()
 
 	detailedMap := DetailedMap{
 		All:      EnvironmentVariableMap{},
 		BySource: BySource{},
 	}
 
-	detailedMap.BySource.Explicit = fromKeys(all, keys)
+	detailedMap.BySource.Explicit = FromKeys(all, keys)
 	detailedMap.All.Merge(detailedMap.BySource.Explicit)
 
 	// Create an excluder function to pass to matcher.