1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
|
import pytest
@pytest.fixture
def stepwise_testdir(testdir):
# Rather than having to modify our testfile between tests, we introduce
# a flag for whether or not the second test should fail.
testdir.makeconftest(
"""
def pytest_addoption(parser):
group = parser.getgroup('general')
group.addoption('--fail', action='store_true', dest='fail')
group.addoption('--fail-last', action='store_true', dest='fail_last')
"""
)
# Create a simple test suite.
testdir.makepyfile(
test_a="""
def test_success_before_fail():
assert 1
def test_fail_on_flag(request):
assert not request.config.getvalue('fail')
def test_success_after_fail():
assert 1
def test_fail_last_on_flag(request):
assert not request.config.getvalue('fail_last')
def test_success_after_last_fail():
assert 1
"""
)
testdir.makepyfile(
test_b="""
def test_success():
assert 1
"""
)
# customize cache directory so we don't use the tox's cache directory, which makes tests in this module flaky
testdir.makeini(
"""
[pytest]
cache_dir = .cache
"""
)
return testdir
@pytest.fixture
def error_testdir(testdir):
testdir.makepyfile(
test_a="""
def test_error(nonexisting_fixture):
assert 1
def test_success_after_fail():
assert 1
"""
)
return testdir
@pytest.fixture
def broken_testdir(testdir):
testdir.makepyfile(
working_testfile="def test_proper(): assert 1", broken_testfile="foobar"
)
return testdir
def _strip_resource_warnings(lines):
# Strip unreliable ResourceWarnings, so no-output assertions on stderr can work.
# (https://github.com/pytest-dev/pytest/issues/5088)
return [
x
for x in lines
if not x.startswith(("Exception ignored in:", "ResourceWarning"))
]
def test_run_without_stepwise(stepwise_testdir):
result = stepwise_testdir.runpytest("-v", "--strict-markers", "--fail")
result.stdout.fnmatch_lines(["*test_success_before_fail PASSED*"])
result.stdout.fnmatch_lines(["*test_fail_on_flag FAILED*"])
result.stdout.fnmatch_lines(["*test_success_after_fail PASSED*"])
def test_fail_and_continue_with_stepwise(stepwise_testdir):
# Run the tests with a failing second test.
result = stepwise_testdir.runpytest(
"-v", "--strict-markers", "--stepwise", "--fail"
)
assert _strip_resource_warnings(result.stderr.lines) == []
stdout = result.stdout.str()
# Make sure we stop after first failing test.
assert "test_success_before_fail PASSED" in stdout
assert "test_fail_on_flag FAILED" in stdout
assert "test_success_after_fail" not in stdout
# "Fix" the test that failed in the last run and run it again.
result = stepwise_testdir.runpytest("-v", "--strict-markers", "--stepwise")
assert _strip_resource_warnings(result.stderr.lines) == []
stdout = result.stdout.str()
# Make sure the latest failing test runs and then continues.
assert "test_success_before_fail" not in stdout
assert "test_fail_on_flag PASSED" in stdout
assert "test_success_after_fail PASSED" in stdout
def test_run_with_skip_option(stepwise_testdir):
result = stepwise_testdir.runpytest(
"-v",
"--strict-markers",
"--stepwise",
"--stepwise-skip",
"--fail",
"--fail-last",
)
assert _strip_resource_warnings(result.stderr.lines) == []
stdout = result.stdout.str()
# Make sure first fail is ignore and second fail stops the test run.
assert "test_fail_on_flag FAILED" in stdout
assert "test_success_after_fail PASSED" in stdout
assert "test_fail_last_on_flag FAILED" in stdout
assert "test_success_after_last_fail" not in stdout
def test_fail_on_errors(error_testdir):
result = error_testdir.runpytest("-v", "--strict-markers", "--stepwise")
assert _strip_resource_warnings(result.stderr.lines) == []
stdout = result.stdout.str()
assert "test_error ERROR" in stdout
assert "test_success_after_fail" not in stdout
def test_change_testfile(stepwise_testdir):
result = stepwise_testdir.runpytest(
"-v", "--strict-markers", "--stepwise", "--fail", "test_a.py"
)
assert _strip_resource_warnings(result.stderr.lines) == []
stdout = result.stdout.str()
assert "test_fail_on_flag FAILED" in stdout
# Make sure the second test run starts from the beginning, since the
# test to continue from does not exist in testfile_b.
result = stepwise_testdir.runpytest(
"-v", "--strict-markers", "--stepwise", "test_b.py"
)
assert _strip_resource_warnings(result.stderr.lines) == []
stdout = result.stdout.str()
assert "test_success PASSED" in stdout
@pytest.mark.parametrize("broken_first", [True, False])
def test_stop_on_collection_errors(broken_testdir, broken_first):
"""Stop during collection errors. Broken test first or broken test last
actually surfaced a bug (#5444), so we test both situations."""
files = ["working_testfile.py", "broken_testfile.py"]
if broken_first:
files.reverse()
result = broken_testdir.runpytest("-v", "--strict-markers", "--stepwise", *files)
result.stdout.fnmatch_lines("*error during collection*")
def test_xfail_handling(testdir, monkeypatch):
"""Ensure normal xfail is ignored, and strict xfail interrupts the session in sw mode
(#5547)
"""
monkeypatch.setattr("sys.dont_write_bytecode", True)
contents = """
import pytest
def test_a(): pass
@pytest.mark.xfail(strict={strict})
def test_b(): assert {assert_value}
def test_c(): pass
def test_d(): pass
"""
testdir.makepyfile(contents.format(assert_value="0", strict="False"))
result = testdir.runpytest("--sw", "-v")
result.stdout.fnmatch_lines(
[
"*::test_a PASSED *",
"*::test_b XFAIL *",
"*::test_c PASSED *",
"*::test_d PASSED *",
"* 3 passed, 1 xfailed in *",
]
)
testdir.makepyfile(contents.format(assert_value="1", strict="True"))
result = testdir.runpytest("--sw", "-v")
result.stdout.fnmatch_lines(
[
"*::test_a PASSED *",
"*::test_b FAILED *",
"* Interrupted*",
"* 1 failed, 1 passed in *",
]
)
testdir.makepyfile(contents.format(assert_value="0", strict="True"))
result = testdir.runpytest("--sw", "-v")
result.stdout.fnmatch_lines(
[
"*::test_b XFAIL *",
"*::test_c PASSED *",
"*::test_d PASSED *",
"* 2 passed, 1 deselected, 1 xfailed in *",
]
)
|
