Adding Failed exception to manage maxfail behavior

Author: jespino

_pytest/main.py

@@ -102,6 +102,8 @@ def wrap_session(config, doit):
             session.exitstatus = doit(config, session) or 0
         except UsageError:
             raise
+        except Failed:
+            session.exitstatus = EXIT_TESTSFAILED
         except KeyboardInterrupt:
             excinfo = _pytest._code.ExceptionInfo()
             if initstate < 2 and isinstance(excinfo.value, exit.Exception):
@@ -159,6 +161,8 @@ def pytest_runtestloop(session):
     for i, item in enumerate(session.items):
         nextitem = session.items[i + 1] if i + 1 < len(session.items) else None
         item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
+        if session.shouldfail:
+            raise session.Failed(session.shouldfail)
         if session.shouldstop:
             raise session.Interrupted(session.shouldstop)
     return True
@@ -564,15 +568,21 @@ class Interrupted(KeyboardInterrupt):
     __module__ = 'builtins'  # for py3
 
 
+class Failed(Exception):
+    """ signals an stop as failed test run. """
+
+
 class Session(FSCollector):
     Interrupted = Interrupted
+    Failed = Failed
 
     def __init__(self, config):
         FSCollector.__init__(self, config.rootdir, parent=None,
                              config=config, session=self)
         self.testsfailed = 0
         self.testscollected = 0
         self.shouldstop = False
+        self.shouldfail = False
         self.trace = config.trace.root.get("collection")
         self._norecursepatterns = config.getini("norecursedirs")
         self.startdir = py.path.local()
@@ -583,6 +593,8 @@ def _makeid(self):
 
     @hookimpl(tryfirst=True)
     def pytest_collectstart(self):
+        if self.shouldfail:
+            raise self.Failed(self.shouldfail)
         if self.shouldstop:
             raise self.Interrupted(self.shouldstop)
 
@@ -592,7 +604,7 @@ def pytest_runtest_logreport(self, report):
             self.testsfailed += 1
             maxfail = self.config.getvalue("maxfail")
             if maxfail and self.testsfailed >= maxfail:
-                self.shouldstop = "stopping after %d failures" % (
+                self.shouldfail = "stopping after %d failures" % (
                     self.testsfailed)
     pytest_collectreport = pytest_runtest_logreport
 

changelog/2845.bugfix

@@ -0,0 +1 @@
+Change return value of pytest command when maxfail is reached from 2 to 1.

testing/test_collection.py

@@ -767,12 +767,11 @@ def test_exit_on_collection_with_maxfail_smaller_than_n_errors(testdir):
     testdir.makepyfile(**COLLECTION_ERROR_PY_FILES)
 
     res = testdir.runpytest("--maxfail=1")
-    assert res.ret == 2
+    assert res.ret == 1
 
     res.stdout.fnmatch_lines([
         "*ERROR collecting test_02_import_error.py*",
         "*No module named *asdfa*",
-        "*Interrupted: stopping after 1 failures*",
     ])
 
     assert 'test_03' not in res.stdout.str()
@@ -824,10 +823,9 @@ def test_continue_on_collection_errors_maxfail(testdir):
     testdir.makepyfile(**COLLECTION_ERROR_PY_FILES)
 
     res = testdir.runpytest("--continue-on-collection-errors", "--maxfail=3")
-    assert res.ret == 2
+    assert res.ret == 1
 
     res.stdout.fnmatch_lines([
         "collected 2 items / 2 errors",
-        "*Interrupted: stopping after 3 failures*",
         "*1 failed, 2 error*",
     ])

testing/test_terminal.py

@@ -757,7 +757,6 @@ def test_3():
         result.stdout.fnmatch_lines([
             "*def test_1():*",
             "*def test_2():*",
-            "*!! Interrupted: stopping after 2 failures*!!*",
             "*2 failed*",
         ])