Introduce notion of a test group, use for CA tests

Grouping our tests should make it easier to understand the intent than
one long poorly-arranged list. It also is convenient for running just
the tests for a specific component when working on that component.

We need at least one test group so this isn't dead code; I decided to
collect the tests for the `ca-derivations` and `dynamic-derivations`
experimental features in groups. Do
```bash
make ca.test-group -jN
```
and
```bash
make dyn-drv.test-group -jN
```
to try running just them.

I originally did this as part of NixOS#8397 for being able to just the local
overlay store alone. I am PRing it separately now so we can separate
general infra from new features.

Co-authored-by: Valentin Gagarin <[email protected]>

Makefile

@@ -27,6 +27,8 @@ makefiles += \
   src/libstore/tests/local.mk \
   src/libexpr/tests/local.mk \
   tests/local.mk \
+  tests/ca/local.mk \
+  tests/dyn-drv/local.mk \
   tests/test-libstoreconsumer/local.mk \
   tests/plugins/local.mk
 else

doc/manual/src/contributing/testing.md

@@ -14,6 +14,8 @@ You can run the whole testsuite with `make check`, or the tests for a specific c
 The functional tests reside under the `tests` directory and are listed in `tests/local.mk`.
 Each test is a bash script.
 
+### Running the whole test suite
+
 The whole test suite can be run with:
 
 ```shell-session
@@ -23,6 +25,33 @@ ran test tests/bar.sh... [PASS]
 ...
 ```
 
+### Grouping tests
+
+Sometimes it is useful to group related tests so they can be easily run together without running the entire test suite.
+Each test group is in a subdirectory of `tests`.
+For example, `tests/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs.
+
+That test group can be run like this:
+
+```shell-session
+$ make ca.test-group -j50
+ran test tests/ca/nix-run.sh... [PASS]
+ran test tests/ca/import-derivation.sh... [PASS]
+...
+```
+
+The test group is defined in Make like this:
+```makefile
+$(test-group-name)-tests := \
+  $(d)/test0.sh \
+  $(d)/test1.sh \
+  ...
+
+install-tests-groups += $(test-group-name)
+```
+
+### Running individual tests
+
 Individual tests can be run with `make`:
 
 ```shell-session

mk/lib.mk

@@ -10,6 +10,7 @@ bin-scripts :=
 noinst-scripts :=
 man-pages :=
 install-tests :=
+install-tests-groups :=
 
 ifdef HOST_OS
   HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS)))
@@ -121,7 +122,16 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b
 $(foreach script, $(bin-scripts), $(eval programs-list += $(script)))
 $(foreach script, $(noinst-scripts), $(eval programs-list += $(script)))
 $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template))))
-$(foreach test, $(install-tests), $(eval $(call run-install-test,$(test))))
+$(foreach test, $(install-tests), \
+  $(eval $(call run-install-test,$(test))) \
+  $(eval installcheck: $(test).test))
+$(foreach test-group, $(install-tests-groups), \
+  $(eval $(call run-install-test-group,$(test-group))) \
+  $(eval installcheck: $(test-group).test-group) \
+  $(foreach test, $($(test-group)-tests), \
+    $(eval $(call run-install-test,$(test))) \
+    $(eval $(test-group).test-group: $(test).test)))
+
 $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file))))))
 
 
@@ -151,6 +161,14 @@ ifdef libs-list
 	@echo "The following libraries can be built:"
 	@echo ""
 	@for i in $(libs-list); do echo "  $$i"; done
+endif
+ifdef install-tests-groups
+	@echo ""
+	@echo "The following groups of functional tests can be run:"
+	@echo ""
+	@for i in $(install-tests-groups); do echo "  $$i.test-group"; done
+	@echo ""
+	@echo "(installcheck includes tests in test groups too.)"
 endif
 	@echo ""
 	@echo "The following variables control the build:"

mk/tests.mk

@@ -4,8 +4,6 @@ test-deps =
 
 define run-install-test
 
-  installcheck: $1.test
-
   .PHONY: $1.test
   $1.test: $1 $(test-deps)
 	@env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null
@@ -16,6 +14,12 @@ define run-install-test
 
 endef
 
+define run-install-test-group
+
+  .PHONY: $1.test-group
+
+endef
+
 .PHONY: check installcheck
 
 print-top-help += \

tests/ca/local.mk

@@ -0,0 +1,27 @@
+ca-tests := \
+  $(d)/build-with-garbage-path.sh \
+  $(d)/build.sh \
+  $(d)/concurrent-builds.sh \
+  $(d)/derivation-json.sh \
+  $(d)/duplicate-realisation-in-closure.sh \
+  $(d)/gc.sh \
+  $(d)/import-derivation.sh \
+  $(d)/new-build-cmd.sh \
+  $(d)/nix-copy.sh \
+  $(d)/nix-run.sh \
+  $(d)/nix-shell.sh \
+  $(d)/post-hook.sh \
+  $(d)/recursive.sh \
+  $(d)/repl.sh \
+  $(d)/selfref-gc.sh \
+  $(d)/signatures.sh \
+  $(d)/substitute.sh \
+  $(d)/why-depends.sh
+
+install-tests-groups += ca
+
+clean-files += \
+  $(d)/config.nix
+
+test-deps += \
+  tests/ca/config.nix

tests/dyn-drv/local.mk

@@ -0,0 +1,11 @@
+dyn-drv-tests := \
+  $(d)/text-hashed-output.sh \
+  $(d)/recursive-mod-json.sh
+
+install-tests-groups += dyn-drv
+
+clean-files += \
+  $(d)/config.nix
+
+test-deps += \
+  tests/dyn-drv/config.nix

tests/local.mk

@@ -14,7 +14,6 @@ nix_tests = \
   flakes/absolute-paths.sh \
   flakes/build-paths.sh \
   flakes/flake-in-submodule.sh \
-  ca/gc.sh \
   gc.sh \
   nix-collect-garbage-d.sh \
   remote-store.sh \
@@ -28,8 +27,6 @@ nix_tests = \
   user-envs-migration.sh \
   binary-cache.sh \
   multiple-outputs.sh \
-  ca/build.sh \
-  ca/new-build-cmd.sh \
   nix-build.sh \
   gc-concurrent.sh \
   repair.sh \
@@ -47,24 +44,17 @@ nix_tests = \
   referrers.sh \
   optimise-store.sh \
   substitute-with-invalid-ca.sh \
-  ca/concurrent-builds.sh \
   signing.sh \
-  ca/build-with-garbage-path.sh \
   hash.sh \
   gc-non-blocking.sh \
   check.sh \
-  ca/substitute.sh \
   nix-shell.sh \
-  ca/signatures.sh \
-  ca/nix-shell.sh \
-  ca/nix-copy.sh \
   check-refs.sh \
   build-remote-input-addressed.sh \
   secure-drv-outputs.sh \
   restricted.sh \
   fetchGitSubmodules.sh \
   flakes/search-root.sh \
-  ca/duplicate-realisation-in-closure.sh \
   readfile-context.sh \
   nix-channel.sh \
   recursive.sh \
@@ -80,10 +70,7 @@ nix_tests = \
   nar-access.sh \
   pure-eval.sh \
   eval.sh \
-  ca/post-hook.sh \
   repl.sh \
-  ca/repl.sh \
-  ca/recursive.sh \
   binary-cache-build-remote.sh \
   search.sh \
   logging.sh \
@@ -109,13 +96,8 @@ nix_tests = \
   fmt.sh \
   eval-store.sh \
   why-depends.sh \
-  ca/why-depends.sh \
   derivation-json.sh \
-  ca/derivation-json.sh \
   import-derivation.sh \
-  ca/import-derivation.sh \
-  dyn-drv/text-hashed-output.sh \
-  dyn-drv/recursive-mod-json.sh \
   nix_path.sh \
   case-hack.sh \
   placeholders.sh \
@@ -124,8 +106,7 @@ nix_tests = \
   build.sh \
   build-delete.sh \
   output-normalization.sh \
-  ca/nix-run.sh \
-  selfref-gc.sh ca/selfref-gc.sh \
+  selfref-gc.sh \
   db-migration.sh \
   bash-profile.sh \
   pass-as-file.sh \
@@ -150,16 +131,12 @@ install-tests += $(foreach x, $(nix_tests), $(d)/$(x))
 
 clean-files += \
   $(d)/common/vars-and-functions.sh \
-  $(d)/config.nix \
-  $(d)/ca/config.nix \
-  $(d)/dyn-drv/config.nix
+  $(d)/config.nix
 
 test-deps += \
   tests/common/vars-and-functions.sh \
   tests/config.nix \
-  tests/ca/config.nix \
-  tests/test-libstoreconsumer/test-libstoreconsumer \
-  tests/dyn-drv/config.nix
+  tests/test-libstoreconsumer/test-libstoreconsumer
 
 ifeq ($(BUILD_SHARED_LIBS), 1)
   test-deps += tests/plugins/libplugintest.$(SO_EXT)