forked from nedbat/coveragepy
-
Notifications
You must be signed in to change notification settings - Fork 0
/
test_goldtest.py
179 lines (143 loc) · 6.98 KB
/
test_goldtest.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
# For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt
"""Tests of the helpers in goldtest.py"""
from __future__ import annotations
import os.path
import re
import pytest
from tests.coveragetest import CoverageTest, TESTS_DIR
from tests.goldtest import compare, gold_path
from tests.goldtest import contains, contains_any, contains_rx, doesnt_contain
from tests.helpers import os_sep, re_line, remove_tree
GOOD_GETTY = """\
Four score and seven years ago our fathers brought forth upon this continent, a
new nation, conceived in Liberty, and dedicated to the proposition that all men
are created equal.
11/19/9999, Gettysburg, Pennsylvania
"""
BAD_GETTY = """\
Five score and seven years ago our fathers brought forth upon this continent, a
new nation, conceived in Liberty, and dedicated to the proposition that all men
are created equal.
333/4444/55555, Gabcdef, Pennsylvania
"""
SCRUBS = [
# Numbers don't matter when comparing.
(r'\d+', 'D'),
(r'G\w+', 'Gxxx'),
]
def path_regex(path: str) -> str:
"""Convert a file path into a regex that will match that path on any OS."""
return re.sub(r"[/\\]", r"[/\\\\]", path.replace(".", "[.]"))
ACTUAL_DIR = os.path.join(TESTS_DIR, "actual/testing")
ACTUAL_GETTY_FILE = os.path.join(ACTUAL_DIR, "getty/gettysburg.txt")
GOLD_GETTY_FILE = os.path.join(TESTS_DIR, "gold/testing/getty/gettysburg.txt")
GOLD_GETTY_FILE_RX = path_regex(GOLD_GETTY_FILE)
GOLD_PATH_RX = path_regex("/tests/gold/testing/getty/gettysburg.txt")
OUT_PATH_RX = path_regex("out/gettysburg.txt")
class CompareTest(CoverageTest):
"""Tests of goldtest.py:compare()"""
def setUp(self) -> None:
super().setUp()
self.addCleanup(remove_tree, ACTUAL_DIR)
def test_good(self) -> None:
self.make_file("out/gettysburg.txt", GOOD_GETTY)
compare(gold_path("testing/getty"), "out", scrubs=SCRUBS)
self.assert_doesnt_exist(ACTUAL_GETTY_FILE)
def test_bad(self) -> None:
self.make_file("out/gettysburg.txt", BAD_GETTY)
# compare() raises an assertion.
msg = fr"Files differ: .*{GOLD_PATH_RX} != {OUT_PATH_RX}"
with pytest.raises(AssertionError, match=msg):
compare(gold_path("testing/getty"), "out", scrubs=SCRUBS)
# Stdout has a description of the diff. The diff shows the scrubbed content.
stdout = self.stdout()
assert "- Four score" in stdout
assert "+ Five score" in stdout
assert re_line(fr"^:::: diff '.*{GOLD_PATH_RX}' and '{OUT_PATH_RX}'", stdout)
assert re_line(fr"^:::: end diff '.*{GOLD_PATH_RX}' and '{OUT_PATH_RX}'", stdout)
assert (
os_sep(f"Saved actual output to '{ACTUAL_GETTY_FILE}': see tests/gold/README.rst")
in os_sep(stdout)
)
assert " D/D/D, Gxxx, Pennsylvania" in stdout
# The actual file was saved.
with open(ACTUAL_GETTY_FILE) as f:
saved = f.read()
assert saved == BAD_GETTY
def test_good_needs_scrubs(self) -> None:
# Comparing the "good" result without scrubbing the variable parts will fail.
self.make_file("out/gettysburg.txt", GOOD_GETTY)
# compare() raises an assertion.
msg = fr"Files differ: .*{GOLD_PATH_RX} != {OUT_PATH_RX}"
with pytest.raises(AssertionError, match=msg):
compare(gold_path("testing/getty"), "out")
stdout = self.stdout()
assert "- 11/19/1863, Gettysburg, Pennsylvania" in stdout
assert "+ 11/19/9999, Gettysburg, Pennsylvania" in stdout
def test_actual_extra(self) -> None:
self.make_file("out/gettysburg.txt", GOOD_GETTY)
self.make_file("out/another.more", "hi")
# Extra files in the output are ok with actual_extra=True.
compare(gold_path("testing/getty"), "out", scrubs=SCRUBS, actual_extra=True)
# But not without it:
# (test output is in files like /tmp/pytest-of-user/pytest-0/popen-gw3/t76/out)
msg = r"Files in .*[/\\]t\d+[/\\]out only: \['another.more'\]"
with pytest.raises(AssertionError, match=msg):
compare(gold_path("testing/getty"), "out", scrubs=SCRUBS)
self.assert_exists(os.path.join(TESTS_DIR, "actual/testing/getty/another.more"))
# But only the files matching the file_pattern are considered.
compare(gold_path("testing/getty"), "out", file_pattern="*.txt", scrubs=SCRUBS)
def test_xml_good(self) -> None:
self.make_file("out/output.xml", """\
<?xml version="1.0" ?>
<the_root c="three" b="222" a="one">
<also z="nine" x="seven" y="888">
Goodie
</also>
</the_root>
""")
compare(gold_path("testing/xml"), "out", scrubs=SCRUBS)
def test_xml_bad(self) -> None:
self.make_file("out/output.xml", """\
<?xml version="1.0" ?>
<the_root c="nine" b="2" a="one">
<also z="three" x="seven" y="8">
Goodbye
</also>
</the_root>
""")
# compare() raises an exception.
gold_rx = path_regex(gold_path("testing/xml/output.xml"))
out_rx = path_regex("out/output.xml")
msg = fr"Files differ: .*{gold_rx} != {out_rx}"
with pytest.raises(AssertionError, match=msg):
compare(gold_path("testing/xml"), "out", scrubs=SCRUBS)
# Stdout has a description of the diff. The diff shows the
# canonicalized and scrubbed content.
stdout = self.stdout()
assert '- <the_root a="one" b="D" c="three">' in stdout
assert '+ <the_root a="one" b="D" c="nine">' in stdout
class ContainsTest(CoverageTest):
"""Tests of the various "contains" functions in goldtest.py"""
run_in_temp_dir = False
def test_contains(self) -> None:
contains(GOLD_GETTY_FILE, "Four", "fathers", "dedicated")
msg = fr"Missing content in {GOLD_GETTY_FILE_RX}: 'xyzzy'"
with pytest.raises(AssertionError, match=msg):
contains(GOLD_GETTY_FILE, "Four", "fathers", "xyzzy", "dedicated")
def test_contains_rx(self) -> None:
contains_rx(GOLD_GETTY_FILE, r"Fo.r", r"f[abc]thers", "dedi[cdef]ated")
msg = fr"Missing regex in {GOLD_GETTY_FILE_RX}: r'm\[opq\]thers'"
with pytest.raises(AssertionError, match=msg):
contains_rx(GOLD_GETTY_FILE, r"Fo.r", r"m[opq]thers")
def test_contains_any(self) -> None:
contains_any(GOLD_GETTY_FILE, "Five", "Four", "Three")
msg = fr"Missing content in {GOLD_GETTY_FILE_RX}: 'One' \[1 of 3\]"
with pytest.raises(AssertionError, match=msg):
contains_any(GOLD_GETTY_FILE, "One", "Two", "Three")
def test_doesnt_contain(self) -> None:
doesnt_contain(GOLD_GETTY_FILE, "One", "Two", "Three")
msg = fr"Forbidden content in {GOLD_GETTY_FILE_RX}: 'Four'"
with pytest.raises(AssertionError, match=msg):
doesnt_contain(GOLD_GETTY_FILE, "Three", "Four", "Five")