refactor cli tests

Author: tabergma

tests/cli/conftest.py

@@ -1,8 +1,11 @@
 from typing import Callable
 import pytest
+import shutil
 import os
 from _pytest.pytester import Testdir, RunResult
 
+from rasa.utils.io import write_yaml_file
+
 
 @pytest.fixture
 def run(testdir: Testdir) -> Callable[..., RunResult]:
@@ -22,13 +25,60 @@ def do_run(*args, stdin):
     return do_run
 
 
+@pytest.fixture
+def run_in_default_project_without_models(testdir: Testdir) -> Callable[..., RunResult]:
+    os.environ["LOG_LEVEL"] = "ERROR"
+
+    _set_up_initial_project(testdir)
+
+    def do_run(*args):
+        args = ["rasa"] + list(args)
+        return testdir.run(*args)
+
+    return do_run
+
+
 @pytest.fixture
 def run_in_default_project(testdir: Testdir) -> Callable[..., RunResult]:
     os.environ["LOG_LEVEL"] = "ERROR"
-    testdir.run("rasa", "init", "--no-prompt")
+
+    _set_up_initial_project(testdir)
+
+    testdir.run("rasa", "train")
 
     def do_run(*args):
         args = ["rasa"] + list(args)
         return testdir.run(*args)
 
     return do_run
+
+
+def _set_up_initial_project(testdir: Testdir):
+    # copy initial project files
+    testdir.copy_example("rasa/cli/initial_project/actions.py")
+    testdir.copy_example("rasa/cli/initial_project/credentials.yml")
+    testdir.copy_example("rasa/cli/initial_project/domain.yml")
+    testdir.copy_example("rasa/cli/initial_project/endpoints.yml")
+    testdir.mkdir("data")
+    testdir.copy_example("rasa/cli/initial_project/data")
+    testdir.run("mv", "nlu.md", "data/nlu.md")
+    testdir.run("mv", "stories.md", "data/stories.md")
+
+    # create a config file
+    # for the cli test the resulting model is not important, use components that are
+    # fast to train
+    write_yaml_file(
+        {
+            "language": "en",
+            "pipeline": [
+                {"name": "WhitespaceTokenizer"},
+                {"name": "CountVectorsFeaturizer"},
+                {"name": "KeywordIntentClassifier"},
+            ],
+            "policies": [
+                {"name": "MappingPolicy"},
+                {"name": "MemoizationPolicy", "max_history": 5},
+            ],
+        },
+        "config.yml",
+    )

tests/cli/test_rasa_data.py

@@ -6,8 +6,10 @@
 from rasa.cli import data
 
 
-def test_data_split_nlu(run_in_default_project: Callable[..., RunResult]):
-    run_in_default_project(
+def test_data_split_nlu(
+    run_in_default_project_without_models: Callable[..., RunResult]
+):
+    run_in_default_project_without_models(
         "data", "split", "nlu", "-u", "data/nlu.md", "--training-fraction", "0.75"
     )
 
@@ -16,8 +18,10 @@ def test_data_split_nlu(run_in_default_project: Callable[..., RunResult]):
     assert os.path.exists(os.path.join("train_test_split", "training_data.md"))
 
 
-def test_data_convert_nlu(run_in_default_project: Callable[..., RunResult]):
-    run_in_default_project(
+def test_data_convert_nlu(
+    run_in_default_project_without_models: Callable[..., RunResult]
+):
+    run_in_default_project_without_models(
         "data",
         "convert",
         "nlu",

tests/cli/test_rasa_run.py

@@ -4,12 +4,13 @@
 from _pytest.pytester import RunResult
 
 
-def test_run_does_not_start(run_in_default_project: Callable[..., RunResult]):
+def test_run_does_not_start(
+    run_in_default_project_without_models: Callable[..., RunResult]
+):
     os.remove("domain.yml")
-    shutil.rmtree("models")
 
     # the server should not start as no model is configured
-    output = run_in_default_project("run")
+    output = run_in_default_project_without_models("run")
 
     assert "No model found." in output.outlines[0]
 

tests/cli/test_rasa_test.py

@@ -59,10 +59,18 @@ def test_test_nlu_cross_validation(run_in_default_project: Callable[..., RunResu
 
 
 def test_test_nlu_comparison(run_in_default_project: Callable[..., RunResult]):
-    copyfile("config.yml", "nlu-config.yml")
+    copyfile("config.yml", "config-1.yml")
 
     run_in_default_project(
-        "test", "nlu", "-c", "config.yml", "nlu-config.yml", "--run", "2"
+        "test",
+        "nlu",
+        "-config",
+        "config.yml",
+        "config-1.yml",
+        "--run",
+        "2",
+        "-percentages",
+        "75",
     )
 
     assert os.path.exists("results/run_1")
@@ -106,6 +114,7 @@ def test_test_core_comparison_after_train(
         },
         "config_2.yml",
     )
+
     run_in_default_project(
         "train",
         "core",

tests/cli/test_rasa_train.py

@@ -18,10 +18,10 @@
 import rasa.utils.io as io_utils
 
 
-def test_train(run_in_default_project: Callable[..., RunResult]):
+def test_train(run_in_default_project_without_models: Callable[..., RunResult]):
     temp_dir = os.getcwd()
 
-    run_in_default_project(
+    run_in_default_project_without_models(
         "train",
         "-c",
         "config.yml",
@@ -48,10 +48,12 @@ def test_train(run_in_default_project: Callable[..., RunResult]):
     )
 
 
-def test_train_persist_nlu_data(run_in_default_project: Callable[..., RunResult]):
+def test_train_persist_nlu_data(
+    run_in_default_project_without_models: Callable[..., RunResult]
+):
     temp_dir = os.getcwd()
 
-    run_in_default_project(
+    run_in_default_project_without_models(
         "train",
         "-c",
         "config.yml",
@@ -79,7 +81,9 @@ def test_train_persist_nlu_data(run_in_default_project: Callable[..., RunResult]
     )
 
 
-def test_train_core_compare(run_in_default_project: Callable[..., RunResult]):
+def test_train_core_compare(
+    run_in_default_project_without_models: Callable[..., RunResult]
+):
     temp_dir = os.getcwd()
 
     io_utils.write_yaml_file(
@@ -100,7 +104,7 @@ def test_train_core_compare(run_in_default_project: Callable[..., RunResult]):
         "config_2.yml",
     )
 
-    run_in_default_project(
+    run_in_default_project_without_models(
         "train",
         "core",
         "-c",
@@ -132,11 +136,11 @@ def test_train_core_compare(run_in_default_project: Callable[..., RunResult]):
 
 
 def test_train_no_domain_exists(
-    run_in_default_project: Callable[..., RunResult]
+    run_in_default_project_without_models: Callable[..., RunResult]
 ) -> None:
 
     os.remove("domain.yml")
-    run_in_default_project(
+    run_in_default_project_without_models(
         "train",
         "-c",
         "config.yml",
@@ -191,38 +195,36 @@ def test_train_force(run_in_default_project):
     assert len(files) == 2
 
 
-def test_train_with_only_nlu_data(run_in_default_project):
+def test_train_with_only_nlu_data(run_in_default_project_without_models):
     temp_dir = os.getcwd()
 
     assert os.path.exists(os.path.join(temp_dir, "data/stories.md"))
     os.remove(os.path.join(temp_dir, "data/stories.md"))
-    shutil.rmtree(os.path.join(temp_dir, "models"))
 
-    run_in_default_project("train", "--fixed-model-name", "test-model")
+    run_in_default_project_without_models("train", "--fixed-model-name", "test-model")
 
     assert os.path.exists(os.path.join(temp_dir, "models"))
     files = io_utils.list_files(os.path.join(temp_dir, "models"))
     assert len(files) == 1
     assert os.path.basename(files[0]) == "test-model.tar.gz"
 
 
-def test_train_with_only_core_data(run_in_default_project):
+def test_train_with_only_core_data(run_in_default_project_without_models):
     temp_dir = os.getcwd()
 
     assert os.path.exists(os.path.join(temp_dir, "data/nlu.md"))
     os.remove(os.path.join(temp_dir, "data/nlu.md"))
-    shutil.rmtree(os.path.join(temp_dir, "models"))
 
-    run_in_default_project("train", "--fixed-model-name", "test-model")
+    run_in_default_project_without_models("train", "--fixed-model-name", "test-model")
 
     assert os.path.exists(os.path.join(temp_dir, "models"))
     files = io_utils.list_files(os.path.join(temp_dir, "models"))
     assert len(files) == 1
     assert os.path.basename(files[0]) == "test-model.tar.gz"
 
 
-def test_train_core(run_in_default_project: Callable[..., RunResult]):
-    run_in_default_project(
+def test_train_core(run_in_default_project_without_models: Callable[..., RunResult]):
+    run_in_default_project_without_models(
         "train",
         "core",
         "-c",
@@ -241,10 +243,12 @@ def test_train_core(run_in_default_project: Callable[..., RunResult]):
     assert os.path.isfile("train_rasa_models/rasa-model.tar.gz")
 
 
-def test_train_core_no_domain_exists(run_in_default_project: Callable[..., RunResult]):
+def test_train_core_no_domain_exists(
+    run_in_default_project_without_models: Callable[..., RunResult]
+):
 
     os.remove("domain.yml")
-    run_in_default_project(
+    run_in_default_project_without_models(
         "train",
         "core",
         "--config",
@@ -263,8 +267,8 @@ def test_train_core_no_domain_exists(run_in_default_project: Callable[..., RunRe
     assert not os.path.isfile("train_rasa_models_no_domain/rasa-model.tar.gz")
 
 
-def test_train_nlu(run_in_default_project: Callable[..., RunResult]):
-    run_in_default_project(
+def test_train_nlu(run_in_default_project_without_models: Callable[..., RunResult]):
+    run_in_default_project_without_models(
         "train",
         "nlu",
         "-c",
@@ -289,9 +293,9 @@ def test_train_nlu(run_in_default_project: Callable[..., RunResult]):
 
 
 def test_train_nlu_persist_nlu_data(
-    run_in_default_project: Callable[..., RunResult]
+    run_in_default_project_without_models: Callable[..., RunResult]
 ) -> None:
-    run_in_default_project(
+    run_in_default_project_without_models(
         "train",
         "nlu",
         "-c",