5978 test time configurable algo template path or url (Project-MONAI#…

…5979)

Signed-off-by: Wenqi Li <[email protected]>

Fixes Project-MONAI#5978

### Description
necessary logic for flexible test cases of algo templates auto3dseg

### Types of changes
<!--- Put an `x` in all the boxes that apply, and remove the not
applicable items -->
- [x] Non-breaking change (fix or new feature that would not break
existing functionality).
- [ ] Breaking change (fix or new feature that would cause existing
functionality to change).
- [ ] New tests added to cover the changes.
- [ ] Integration tests passed locally by running `./runtests.sh -f -u
--net --coverage`.
- [ ] Quick tests passed locally by running `./runtests.sh --quick
--unittests --disttests`.
- [ ] In-line docstrings updated.
- [ ] Documentation updated, tested `make html` command in the `docs/`
folder.

---------

Signed-off-by: Wenqi Li <[email protected]>

monai/apps/auto3dseg/bundle_gen.py

@@ -326,7 +326,7 @@ def _download_algos_url(url: str, at_path: str) -> dict[str, dict[str, str]]:
 def _copy_algos_folder(folder, at_path):
     """
     Copies the algorithm templates folder to at_path.
-    Returns a dictionary of of algorithm templates.
+    Returns a dictionary of algorithm templates.
     """
     folder = os.path.abspath(folder)
     at_path = os.path.abspath(at_path)
@@ -342,7 +342,8 @@ def _copy_algos_folder(folder, at_path):
             algos_all[name] = dict(
                 _target_=f"{name}.scripts.algo.{name.capitalize()}Algo", template_path=os.path.join(at_path, name)
             )
-    if len(algos_all) == 0:
+            logger.info(f"{name} -- {algos_all[name]}")
+    if not algos_all:
         raise ValueError(f"Unable to find any algos in {folder}")
 
     return algos_all
@@ -385,9 +386,11 @@ def __init__(
 
             if os.path.isdir(templates_path_or_url):
                 # if a local folder, copy if necessary
+                logger.info(f"BundleGen from directory {templates_path_or_url}")
                 algos_all = _copy_algos_folder(folder=templates_path_or_url, at_path=at_path)
             elif urlparse(templates_path_or_url).scheme in ("http", "https"):
                 # if url, trigger the download and extract process
+                logger.info(f"BundleGen from {templates_path_or_url}")
                 algos_all = _download_algos_url(url=templates_path_or_url, at_path=at_path)
             else:
                 raise ValueError(f"{self.__class__} received invalid templates_path_or_url: {templates_path_or_url}")

tests/test_auto3dseg_ensemble.py

@@ -24,7 +24,13 @@
 from monai.data import create_test_image_3d
 from monai.utils import optional_import, set_determinism
 from monai.utils.enums import AlgoEnsembleKeys
-from tests.utils import SkipIfBeforePyTorchVersion, skip_if_downloading_fails, skip_if_no_cuda, skip_if_quick
+from tests.utils import (
+    SkipIfBeforePyTorchVersion,
+    get_testing_algo_template_path,
+    skip_if_downloading_fails,
+    skip_if_no_cuda,
+    skip_if_quick,
+)
 
 _, has_tb = optional_import("torch.utils.tensorboard", name="SummaryWriter")
 
@@ -120,7 +126,10 @@ def test_ensemble(self) -> None:
 
         with skip_if_downloading_fails():
             bundle_generator = BundleGen(
-                algo_path=work_dir, data_stats_filename=da_output_yaml, data_src_cfg_name=data_src_cfg
+                algo_path=work_dir,
+                data_stats_filename=da_output_yaml,
+                data_src_cfg_name=data_src_cfg,
+                templates_path_or_url=get_testing_algo_template_path(),
             )
         bundle_generator.generate(work_dir, num_fold=1)
         history = bundle_generator.get_history()

tests/test_auto3dseg_hpo.py

@@ -24,7 +24,12 @@
 from monai.bundle.config_parser import ConfigParser
 from monai.data import create_test_image_3d
 from monai.utils import optional_import
-from tests.utils import SkipIfBeforePyTorchVersion, skip_if_downloading_fails, skip_if_no_cuda
+from tests.utils import (
+    SkipIfBeforePyTorchVersion,
+    get_testing_algo_template_path,
+    skip_if_downloading_fails,
+    skip_if_no_cuda,
+)
 
 _, has_tb = optional_import("torch.utils.tensorboard", name="SummaryWriter")
 optuna, has_optuna = optional_import("optuna")
@@ -120,7 +125,10 @@ def setUp(self) -> None:
         ConfigParser.export_config_file(data_src, data_src_cfg)
         with skip_if_downloading_fails():
             bundle_generator = BundleGen(
-                algo_path=work_dir, data_stats_filename=da_output_yaml, data_src_cfg_name=data_src_cfg
+                algo_path=work_dir,
+                data_stats_filename=da_output_yaml,
+                data_src_cfg_name=data_src_cfg,
+                templates_path_or_url=get_testing_algo_template_path(),
             )
         bundle_generator.generate(work_dir, num_fold=1)
 

tests/test_integration_autorunner.py

@@ -23,7 +23,13 @@
 from monai.bundle.config_parser import ConfigParser
 from monai.data import create_test_image_3d
 from monai.utils import optional_import
-from tests.utils import SkipIfBeforePyTorchVersion, skip_if_downloading_fails, skip_if_no_cuda, skip_if_quick
+from tests.utils import (
+    SkipIfBeforePyTorchVersion,
+    get_testing_algo_template_path,
+    skip_if_downloading_fails,
+    skip_if_no_cuda,
+    skip_if_quick,
+)
 
 _, has_tb = optional_import("torch.utils.tensorboard", name="SummaryWriter")
 _, has_nni = optional_import("nni")
@@ -107,7 +113,9 @@ def setUp(self) -> None:
     @skip_if_no_cuda
     def test_autorunner(self) -> None:
         work_dir = os.path.join(self.test_path, "work_dir")
-        runner = AutoRunner(work_dir=work_dir, input=self.data_src_cfg)
+        runner = AutoRunner(
+            work_dir=work_dir, input=self.data_src_cfg, templates_path_or_url=get_testing_algo_template_path()
+        )
         runner.set_training_params(train_param)  # 2 epochs
         runner.set_num_fold(1)
         with skip_if_downloading_fails():
@@ -116,7 +124,9 @@ def test_autorunner(self) -> None:
     @skip_if_no_cuda
     def test_autorunner_ensemble(self) -> None:
         work_dir = os.path.join(self.test_path, "work_dir")
-        runner = AutoRunner(work_dir=work_dir, input=self.data_src_cfg)
+        runner = AutoRunner(
+            work_dir=work_dir, input=self.data_src_cfg, templates_path_or_url=get_testing_algo_template_path()
+        )
         runner.set_training_params(train_param)  # 2 epochs
         runner.set_ensemble_method("AlgoEnsembleBestByFold")
         runner.set_num_fold(1)
@@ -126,7 +136,9 @@ def test_autorunner_ensemble(self) -> None:
     @skip_if_no_cuda
     def test_autorunner_gpu_customization(self) -> None:
         work_dir = os.path.join(self.test_path, "work_dir")
-        runner = AutoRunner(work_dir=work_dir, input=self.data_src_cfg)
+        runner = AutoRunner(
+            work_dir=work_dir, input=self.data_src_cfg, templates_path_or_url=get_testing_algo_template_path()
+        )
         gpu_customization_specs = {
             "universal": {"num_trials": 1, "range_num_images_per_batch": [1, 2], "range_num_sw_batch_size": [1, 2]}
         }
@@ -140,7 +152,13 @@ def test_autorunner_gpu_customization(self) -> None:
     @unittest.skipIf(not has_nni, "nni required")
     def test_autorunner_hpo(self) -> None:
         work_dir = os.path.join(self.test_path, "work_dir")
-        runner = AutoRunner(work_dir=work_dir, input=self.data_src_cfg, hpo=True, ensemble=False)
+        runner = AutoRunner(
+            work_dir=work_dir,
+            input=self.data_src_cfg,
+            hpo=True,
+            ensemble=False,
+            templates_path_or_url=get_testing_algo_template_path(),
+        )
         hpo_param = {
             "num_epochs_per_validation": train_param["num_epochs_per_validation"],
             "num_images_per_batch": train_param["num_images_per_batch"],

tests/test_integration_gpu_customization.py

@@ -24,7 +24,13 @@
 from monai.data import create_test_image_3d
 from monai.utils import optional_import
 from monai.utils.enums import AlgoEnsembleKeys
-from tests.utils import SkipIfBeforePyTorchVersion, skip_if_downloading_fails, skip_if_no_cuda, skip_if_quick
+from tests.utils import (
+    SkipIfBeforePyTorchVersion,
+    get_testing_algo_template_path,
+    skip_if_downloading_fails,
+    skip_if_no_cuda,
+    skip_if_quick,
+)
 
 _, has_tb = optional_import("torch.utils.tensorboard", name="SummaryWriter")
 
@@ -118,7 +124,10 @@ def test_ensemble_gpu_customization(self) -> None:
 
         with skip_if_downloading_fails():
             bundle_generator = BundleGen(
-                algo_path=work_dir, data_stats_filename=da_output_yaml, data_src_cfg_name=data_src_cfg
+                algo_path=work_dir,
+                data_stats_filename=da_output_yaml,
+                data_src_cfg_name=data_src_cfg,
+                templates_path_or_url=get_testing_algo_template_path(),
             )
 
         gpu_customization_specs = {

tests/utils.py

@@ -66,6 +66,16 @@ def testing_data_config(*keys):
     return reduce(operator.getitem, keys, _test_data_config)
 
 
+def get_testing_algo_template_path():
+    """
+    a local folder to the testing algorithm template or a url to the compressed template file.
+    Default to None, which effectively uses bundle_gen's ``default_algo_zip`` path.
+
+    https://github.com/Project-MONAI/MONAI/blob/1.1.0/monai/apps/auto3dseg/bundle_gen.py#L380-L381
+    """
+    return os.environ.get("MONAI_TESTING_ALGO_TEMPLATE", None)
+
+
 def clone(data: NdarrayTensor) -> NdarrayTensor:
     """
     Clone data independent of type.