Fix strict xfail: it should behave exactly like xfail when a test fails

1 files changed, 22 insertions, 3 deletions

diff --git a/testing/test_skipping.py b/testing/test_skipping.py
index 227177eb0..6683aa200 100644
--- a/testing/test_skipping.py
+++ b/testing/test_skipping.py
@@ -127,13 +127,15 @@ class TestEvaluator:
 
 
 class TestXFail:
-    def test_xfail_simple(self, testdir):
+
+    @pytest.mark.parametrize('strict', [True, False])
+    def test_xfail_simple(self, testdir, strict):
         item = testdir.getitem("""
             import pytest
-            @pytest.mark.xfail
+            @pytest.mark.xfail(strict=%s)
             def test_func():
                 assert 0
-        """)
+        """ % strict)
         reports = runtestprotocol(item, log=False)
         assert len(reports) == 3
         callreport = reports[1]
@@ -350,6 +352,23 @@ class TestXFail:
             matchline,
         ])
 
+    def test_strict_sanity(self, testdir):
+        """sanity check for xfail(strict=True): a failing test should behave
+        exactly like a normal xfail.
+        """
+        p = testdir.makepyfile("""
+            import pytest
+            @pytest.mark.xfail(reason='unsupported feature', strict=True)
+            def test_foo():
+                assert 0
+        """)
+        result = testdir.runpytest(p, '-rxX')
+        result.stdout.fnmatch_lines([
+            '*XFAIL*',
+            '*unsupported feature*',
+        ])
+        assert result.ret == 0
+
     @pytest.mark.parametrize('strict', [True, False])
     def test_strict_xfail(self, testdir, strict):
         p = testdir.makepyfile("""