Use OutputFormat in tests instead of strings (semgrep#3051)

semgrep/tests/README.md

@@ -95,12 +95,12 @@ You can add any additional CLI flag you'd like like so:
 
 To call semgrep without the `--json` flag:
 
-```run_semgrep_in_tmp("r2c/python", output_format="normal")```
+```run_semgrep_in_tmp("r2c/python", output_format=OutputFormat.TEXT)```
 
-To call semgrep without the `--junit-xml` flag:
+To call semgrep with the `--junit-xml` flag:
 
-```run_semgrep_in_tmp("r2c/python", output_format="junit-xml")```
+```run_semgrep_in_tmp("r2c/python", output_format=OutputFormat.JUNIT_XML)```
 
-To call semgrep with `--sarif` instead of `--json`:
+To call semgrep with the `--sarif` flag instead of `--json`:
 
-```run_semgrep_in_tmp("r2c/python", output_format="sarif")```
+```run_semgrep_in_tmp("r2c/python", output_format=OutputFormat.SARIF)```

semgrep/tests/conftest.py

@@ -13,6 +13,7 @@
 import pytest
 
 from . import public_repos
+from semgrep.constants import OutputFormat
 
 TESTS_PATH = Path(__file__).parent
 
@@ -61,7 +62,7 @@ def _run_semgrep(
     *,
     target_name: str = "basic",
     options: Optional[List[Union[str, Path]]] = None,
-    output_format: str = "json",
+    output_format: OutputFormat = OutputFormat.JSON,
     stderr: bool = False,
     strict: bool = True,
 ) -> str:
@@ -70,7 +71,7 @@ def _run_semgrep(
     :param config: what to pass as --config's value
     :param target_name: which directory within ./e2e/targets/ to scan
     :param options: additional CLI flags to add
-    :param output_format: which format to use, valid options are normal, json, junit_xml and sarif
+    :param output_format: which format to use
     :param stderr: whether to merge stderr into the returned string
     """
     if options is None:
@@ -88,11 +89,11 @@ def _run_semgrep(
         else:
             options.extend(["--config", config])
 
-    if output_format == "json":
+    if output_format == OutputFormat.JSON:
         options.append("--json")
-    elif output_format == "junit-xml":
+    elif output_format == OutputFormat.JUNIT_XML:
         options.append("--junit-xml")
-    elif output_format == "sarif":
+    elif output_format == OutputFormat.SARIF:
         options.append("--sarif")
 
     output = subprocess.check_output(
@@ -101,7 +102,7 @@ def _run_semgrep(
         stderr=subprocess.STDOUT if stderr else subprocess.PIPE,
     )
 
-    if output_format in {"json", "sarif"} and not stderr:
+    if output_format.is_json() and not stderr:
         output = _clean_output_json(output)
 
     return output

semgrep/tests/e2e/test_check.py

@@ -8,6 +8,7 @@
 import pytest
 
 from semgrep import __VERSION__
+from semgrep.constants import OutputFormat
 
 GITHUB_TEST_GIST_URL = (
     "https://raw.githubusercontent.com/returntocorp/semgrep-rules/develop/template.yaml"
@@ -73,7 +74,8 @@ def test_basic_rule__absolute(run_semgrep_in_tmp, snapshot):
 
 def test_terminal_output(run_semgrep_in_tmp, snapshot):
     snapshot.assert_match(
-        run_semgrep_in_tmp("rules/eqeq.yaml", output_format="normal"), "output.txt"
+        run_semgrep_in_tmp("rules/eqeq.yaml", output_format=OutputFormat.TEXT),
+        "output.txt",
     )
 
 
@@ -85,7 +87,7 @@ def test_multiline(run_semgrep_in_tmp, snapshot):
 
 
 def test_junit_xml_output(run_semgrep_in_tmp, snapshot):
-    output = run_semgrep_in_tmp("rules/eqeq.yaml", output_format="junit-xml")
+    output = run_semgrep_in_tmp("rules/eqeq.yaml", output_format=OutputFormat.JUNIT_XML)
     result = etree_to_dict(cElementTree.XML(output))
 
     filename = snapshot.snapshot_dir / "results.xml"
@@ -96,7 +98,7 @@ def test_junit_xml_output(run_semgrep_in_tmp, snapshot):
 
 def test_sarif_output(run_semgrep_in_tmp, snapshot):
     sarif_output = json.loads(
-        run_semgrep_in_tmp("rules/eqeq.yaml", output_format="sarif")
+        run_semgrep_in_tmp("rules/eqeq.yaml", output_format=OutputFormat.SARIF)
     )
 
     # rules are logically a set so the JSON list's order doesn't matter
@@ -142,7 +144,7 @@ def test_hidden_rule__implicit(run_semgrep_in_tmp, snapshot):
     snapshot.assert_match(excinfo.value.stdout, "error.json")
 
     with pytest.raises(CalledProcessError) as excinfo:
-        run_semgrep_in_tmp("rules/hidden", output_format="normal")
+        run_semgrep_in_tmp("rules/hidden", output_format=OutputFormat.TEXT)
     assert excinfo.value.returncode == 7
     snapshot.assert_match(excinfo.value.stderr, "error.txt")
 
@@ -353,7 +355,7 @@ def test_spacegrep_timeout(run_semgrep_in_tmp, snapshot):
                 "--timeout",
                 "1",
             ],
-            output_format="text",
+            output_format=OutputFormat.TEXT,
             stderr=True,
             strict=False,  # don't fail due to timeout
         ),
@@ -375,7 +377,7 @@ def test_max_memory(run_semgrep_in_tmp, snapshot):
     snapshot.assert_match(
         run_semgrep_in_tmp(
             "rules/long.yaml",
-            output_format="normal",
+            output_format=OutputFormat.TEXT,
             options=["--verbose", "--max-memory", "1"],
             target_name="equivalence",
             strict=False,
@@ -399,7 +401,7 @@ def test_timeout_threshold(run_semgrep_in_tmp, snapshot):
     snapshot.assert_match(
         run_semgrep_in_tmp(
             "rules/multiple-long.yaml",
-            output_format="normal",
+            output_format=OutputFormat.TEXT,
             options=["--verbose", "--timeout", "1", "--timeout-threshold", "1"],
             target_name="equivalence",
             strict=False,
@@ -411,7 +413,7 @@ def test_timeout_threshold(run_semgrep_in_tmp, snapshot):
     snapshot.assert_match(
         run_semgrep_in_tmp(
             "rules/multiple-long.yaml",
-            output_format="normal",
+            output_format=OutputFormat.TEXT,
             options=["--verbose", "--timeout", "1", "--timeout-threshold", "2"],
             target_name="equivalence",
             strict=False,

semgrep/tests/e2e/test_cli_test.py

@@ -1,9 +1,12 @@
+from semgrep.constants import OutputFormat
+
+
 def test_cli_test_basic(run_semgrep_in_tmp, snapshot):
     results = run_semgrep_in_tmp(
         "rules/cli_test/basic/",
         options=["--test"],
         target_name="cli_test/basic/",
-        output_format="json",
+        output_format=OutputFormat.JSON,
     )
     snapshot.assert_match(
         results,
@@ -16,7 +19,7 @@ def test_cli_test_yaml_language(run_semgrep_in_tmp, snapshot):
         "rules/cli_test/language/",
         options=["--test"],
         target_name="cli_test/language/",
-        output_format="json",
+        output_format=OutputFormat.JSON,
     )
     snapshot.assert_match(
         results,
@@ -29,7 +32,7 @@ def test_cli_test_suffixes(run_semgrep_in_tmp, snapshot):
         "rules/cli_test/suffixes/",
         options=["--test"],
         target_name="cli_test/suffixes/",
-        output_format="json",
+        output_format=OutputFormat.JSON,
     )
     snapshot.assert_match(
         results,
@@ -42,7 +45,7 @@ def test_cli_test_multiline_annotations(run_semgrep_in_tmp, snapshot):
         "rules/cli_test/multiple_annotations/",
         options=["--test"],
         target_name="cli_test/multiple_annotations/",
-        output_format="text",
+        output_format=OutputFormat.TEXT,
     )
     snapshot.assert_match(
         results,

semgrep/tests/e2e/test_debugging_json.py

@@ -1,10 +1,13 @@
+from semgrep.constants import OutputFormat
+
+
 def test_debugging_json(run_semgrep_in_tmp, snapshot):
     snapshot.assert_match(
         run_semgrep_in_tmp(
             "rules/eqeq.yaml",
             target_name="basic/stupid.py",
             options=["--debug"],
-            output_format="json",
+            output_format=OutputFormat.JSON,
         ),
         "results.json",
     )

semgrep/tests/e2e/test_rule_parser.py

@@ -4,6 +4,7 @@
 import pytest
 
 from ..conftest import TESTS_PATH
+from semgrep.constants import OutputFormat
 
 syntax_dir = TESTS_PATH / "e2e" / "rules" / "syntax"
 syntax_passes = [f.with_suffix("").name for f in syntax_dir.glob("good*.yaml")]
@@ -47,7 +48,7 @@ def test_rule_parser__failure__error_messages(run_semgrep_in_tmp, snapshot, file
         run_semgrep_in_tmp(
             f"rules/syntax/{filename}.yaml",
             options=["--force-color"],
-            output_format="normal",
+            output_format=OutputFormat.TEXT,
         )
 
     snapshot.assert_match(
@@ -71,6 +72,7 @@ def test_rule_parser_cli_pattern(run_semgrep_in_tmp, snapshot):
     # Check pretty print output
     with pytest.raises(CalledProcessError) as excinfo:
         run_semgrep_in_tmp(
-            options=["-e", "#include<asdf><<>>><$X>", "-l", "c"], output_format="normal"
+            options=["-e", "#include<asdf><<>>><$X>", "-l", "c"],
+            output_format=OutputFormat.TEXT,
         )
     snapshot.assert_match(excinfo.value.stderr, "error.txt")

semgrep/tests/e2e/test_semgrep_core_parse_error.py

@@ -2,6 +2,8 @@
 
 import pytest
 
+from semgrep.constants import OutputFormat
+
 
 @pytest.mark.parametrize(
     "filename",
@@ -15,7 +17,7 @@ def test_rule_parser__failure__error_messages(run_semgrep_in_tmp, snapshot, file
             config="rules/eqeq-python.yaml",
             target_name=f"bad/{filename}",
             options=["--verbose"],
-            output_format="text",
+            output_format=OutputFormat.TEXT,
             stderr=True,
         )
     snapshot.assert_match(excinfo.value.output, "error.txt")