bpo-25894: Always report skipped and failed subtests separately (GH-28082)

* In default mode output separate characters for skipped and failed subtests.
* In verbose mode output separate lines (including description) for skipped
   and failed subtests.
* In verbose mode output test description for errors in test cleanup.

Author: serhiy-storchaka

Lib/unittest/runner.py

@@ -5,6 +5,7 @@
 import warnings
 
 from . import result
+from .case import _SubTest
 from .signals import registerResult
 
 __unittest = True
@@ -40,6 +41,7 @@ def __init__(self, stream, descriptions, verbosity):
         self.showAll = verbosity > 1
         self.dots = verbosity == 1
         self.descriptions = descriptions
+        self._newline = True
 
     def getDescription(self, test):
         doc_first_line = test.shortDescription()
@@ -54,35 +56,63 @@ def startTest(self, test):
             self.stream.write(self.getDescription(test))
             self.stream.write(" ... ")
             self.stream.flush()
+            self._newline = False
+
+    def _write_status(self, test, status):
+        is_subtest = isinstance(test, _SubTest)
+        if is_subtest or self._newline:
+            if not self._newline:
+                self.stream.writeln()
+            if is_subtest:
+                self.stream.write("  ")
+            self.stream.write(self.getDescription(test))
+            self.stream.write(" ... ")
+        self.stream.writeln(status)
+        self._newline = True
+
+    def addSubTest(self, test, subtest, err):
+        if err is not None:
+            if self.showAll:
+                if issubclass(err[0], subtest.failureException):
+                    self._write_status(subtest, "FAIL")
+                else:
+                    self._write_status(subtest, "ERROR")
+            elif self.dots:
+                if issubclass(err[0], subtest.failureException):
+                    self.stream.write('F')
+                else:
+                    self.stream.write('E')
+                self.stream.flush()
+        super(TextTestResult, self).addSubTest(test, subtest, err)
 
     def addSuccess(self, test):
         super(TextTestResult, self).addSuccess(test)
         if self.showAll:
-            self.stream.writeln("ok")
+            self._write_status(test, "ok")
         elif self.dots:
             self.stream.write('.')
             self.stream.flush()
 
     def addError(self, test, err):
         super(TextTestResult, self).addError(test, err)
         if self.showAll:
-            self.stream.writeln("ERROR")
+            self._write_status(test, "ERROR")
         elif self.dots:
             self.stream.write('E')
             self.stream.flush()
 
     def addFailure(self, test, err):
         super(TextTestResult, self).addFailure(test, err)
         if self.showAll:
-            self.stream.writeln("FAIL")
+            self._write_status(test, "FAIL")
         elif self.dots:
             self.stream.write('F')
             self.stream.flush()
 
     def addSkip(self, test, reason):
         super(TextTestResult, self).addSkip(test, reason)
         if self.showAll:
-            self.stream.writeln("skipped {0!r}".format(reason))
+            self._write_status(test, "skipped {0!r}".format(reason))
         elif self.dots:
             self.stream.write("s")
             self.stream.flush()

Lib/unittest/test/test_result.py

@@ -305,37 +305,76 @@ def test_1(self):
         self.assertIs(test_case, subtest)
         self.assertIn("some recognizable failure", formatted_exc)
 
+    def testStackFrameTrimming(self):
+        class Frame(object):
+            class tb_frame(object):
+                f_globals = {}
+        result = unittest.TestResult()
+        self.assertFalse(result._is_relevant_tb_level(Frame))
+
+        Frame.tb_frame.f_globals['__unittest'] = True
+        self.assertTrue(result._is_relevant_tb_level(Frame))
+
+    def testFailFast(self):
+        result = unittest.TestResult()
+        result._exc_info_to_string = lambda *_: ''
+        result.failfast = True
+        result.addError(None, None)
+        self.assertTrue(result.shouldStop)
+
+        result = unittest.TestResult()
+        result._exc_info_to_string = lambda *_: ''
+        result.failfast = True
+        result.addFailure(None, None)
+        self.assertTrue(result.shouldStop)
+
+        result = unittest.TestResult()
+        result._exc_info_to_string = lambda *_: ''
+        result.failfast = True
+        result.addUnexpectedSuccess(None)
+        self.assertTrue(result.shouldStop)
+
+    def testFailFastSetByRunner(self):
+        runner = unittest.TextTestRunner(stream=io.StringIO(), failfast=True)
+        def test(result):
+            self.assertTrue(result.failfast)
+        result = runner.run(test)
+
+
+class Test_TextTestResult(unittest.TestCase):
+    maxDiff = None
+
     def testGetDescriptionWithoutDocstring(self):
         result = unittest.TextTestResult(None, True, 1)
         self.assertEqual(
                 result.getDescription(self),
                 'testGetDescriptionWithoutDocstring (' + __name__ +
-                '.Test_TestResult)')
+                '.Test_TextTestResult)')
 
     def testGetSubTestDescriptionWithoutDocstring(self):
         with self.subTest(foo=1, bar=2):
             result = unittest.TextTestResult(None, True, 1)
             self.assertEqual(
                     result.getDescription(self._subtest),
                     'testGetSubTestDescriptionWithoutDocstring (' + __name__ +
-                    '.Test_TestResult) (foo=1, bar=2)')
+                    '.Test_TextTestResult) (foo=1, bar=2)')
         with self.subTest('some message'):
             result = unittest.TextTestResult(None, True, 1)
             self.assertEqual(
                     result.getDescription(self._subtest),
                     'testGetSubTestDescriptionWithoutDocstring (' + __name__ +
-                    '.Test_TestResult) [some message]')
+                    '.Test_TextTestResult) [some message]')
 
     def testGetSubTestDescriptionWithoutDocstringAndParams(self):
         with self.subTest():
             result = unittest.TextTestResult(None, True, 1)
             self.assertEqual(
                     result.getDescription(self._subtest),
                     'testGetSubTestDescriptionWithoutDocstringAndParams '
-                    '(' + __name__ + '.Test_TestResult) (<subtest>)')
+                    '(' + __name__ + '.Test_TextTestResult) (<subtest>)')
 
     def testGetSubTestDescriptionForFalsyValues(self):
-        expected = 'testGetSubTestDescriptionForFalsyValues (%s.Test_TestResult) [%s]'
+        expected = 'testGetSubTestDescriptionForFalsyValues (%s.Test_TextTestResult) [%s]'
         result = unittest.TextTestResult(None, True, 1)
         for arg in [0, None, []]:
             with self.subTest(arg):
@@ -351,7 +390,7 @@ def testGetNestedSubTestDescriptionWithoutDocstring(self):
                 self.assertEqual(
                         result.getDescription(self._subtest),
                         'testGetNestedSubTestDescriptionWithoutDocstring '
-                        '(' + __name__ + '.Test_TestResult) (baz=2, bar=3, foo=1)')
+                        '(' + __name__ + '.Test_TextTestResult) (baz=2, bar=3, foo=1)')
 
     def testGetDuplicatedNestedSubTestDescriptionWithoutDocstring(self):
         with self.subTest(foo=1, bar=2):
@@ -360,7 +399,7 @@ def testGetDuplicatedNestedSubTestDescriptionWithoutDocstring(self):
                 self.assertEqual(
                         result.getDescription(self._subtest),
                         'testGetDuplicatedNestedSubTestDescriptionWithoutDocstring '
-                        '(' + __name__ + '.Test_TestResult) (baz=3, bar=4, foo=1)')
+                        '(' + __name__ + '.Test_TextTestResult) (baz=3, bar=4, foo=1)')
 
     @unittest.skipIf(sys.flags.optimize >= 2,
                      "Docstrings are omitted with -O2 and above")
@@ -370,7 +409,7 @@ def testGetDescriptionWithOneLineDocstring(self):
         self.assertEqual(
                 result.getDescription(self),
                ('testGetDescriptionWithOneLineDocstring '
-                '(' + __name__ + '.Test_TestResult)\n'
+                '(' + __name__ + '.Test_TextTestResult)\n'
                 'Tests getDescription() for a method with a docstring.'))
 
     @unittest.skipIf(sys.flags.optimize >= 2,
@@ -382,7 +421,7 @@ def testGetSubTestDescriptionWithOneLineDocstring(self):
             self.assertEqual(
                 result.getDescription(self._subtest),
                ('testGetSubTestDescriptionWithOneLineDocstring '
-                '(' + __name__ + '.Test_TestResult) (foo=1, bar=2)\n'
+                '(' + __name__ + '.Test_TextTestResult) (foo=1, bar=2)\n'
                 'Tests getDescription() for a method with a docstring.'))
 
     @unittest.skipIf(sys.flags.optimize >= 2,
@@ -395,7 +434,7 @@ def testGetDescriptionWithMultiLineDocstring(self):
         self.assertEqual(
                 result.getDescription(self),
                ('testGetDescriptionWithMultiLineDocstring '
-                '(' + __name__ + '.Test_TestResult)\n'
+                '(' + __name__ + '.Test_TextTestResult)\n'
                 'Tests getDescription() for a method with a longer '
                 'docstring.'))
 
@@ -410,44 +449,111 @@ def testGetSubTestDescriptionWithMultiLineDocstring(self):
             self.assertEqual(
                 result.getDescription(self._subtest),
                ('testGetSubTestDescriptionWithMultiLineDocstring '
-                '(' + __name__ + '.Test_TestResult) (foo=1, bar=2)\n'
+                '(' + __name__ + '.Test_TextTestResult) (foo=1, bar=2)\n'
                 'Tests getDescription() for a method with a longer '
                 'docstring.'))
 
-    def testStackFrameTrimming(self):
-        class Frame(object):
-            class tb_frame(object):
-                f_globals = {}
-        result = unittest.TestResult()
-        self.assertFalse(result._is_relevant_tb_level(Frame))
-
-        Frame.tb_frame.f_globals['__unittest'] = True
-        self.assertTrue(result._is_relevant_tb_level(Frame))
-
-    def testFailFast(self):
-        result = unittest.TestResult()
-        result._exc_info_to_string = lambda *_: ''
-        result.failfast = True
-        result.addError(None, None)
-        self.assertTrue(result.shouldStop)
-
-        result = unittest.TestResult()
-        result._exc_info_to_string = lambda *_: ''
-        result.failfast = True
-        result.addFailure(None, None)
-        self.assertTrue(result.shouldStop)
-
-        result = unittest.TestResult()
-        result._exc_info_to_string = lambda *_: ''
-        result.failfast = True
-        result.addUnexpectedSuccess(None)
-        self.assertTrue(result.shouldStop)
-
-    def testFailFastSetByRunner(self):
-        runner = unittest.TextTestRunner(stream=io.StringIO(), failfast=True)
-        def test(result):
-            self.assertTrue(result.failfast)
-        result = runner.run(test)
+    class Test(unittest.TestCase):
+        def testSuccess(self):
+            pass
+        def testSkip(self):
+            self.skipTest('skip')
+        def testFail(self):
+            self.fail('fail')
+        def testError(self):
+            raise Exception('error')
+        def testSubTestSuccess(self):
+            with self.subTest('one', a=1):
+                pass
+            with self.subTest('two', b=2):
+                pass
+        def testSubTestMixed(self):
+            with self.subTest('success', a=1):
+                pass
+            with self.subTest('skip', b=2):
+                self.skipTest('skip')
+            with self.subTest('fail', c=3):
+                self.fail('fail')
+            with self.subTest('error', d=4):
+                raise Exception('error')
+
+        tearDownError = None
+        def tearDown(self):
+            if self.tearDownError is not None:
+                raise self.tearDownError
+
+    def _run_test(self, test_name, verbosity, tearDownError=None):
+        stream = io.StringIO()
+        stream = unittest.runner._WritelnDecorator(stream)
+        result = unittest.TextTestResult(stream, True, verbosity)
+        test = self.Test(test_name)
+        test.tearDownError = tearDownError
+        test.run(result)
+        return stream.getvalue()
+
+    def testDotsOutput(self):
+        self.assertEqual(self._run_test('testSuccess', 1), '.')
+        self.assertEqual(self._run_test('testSkip', 1), 's')
+        self.assertEqual(self._run_test('testFail', 1), 'F')
+        self.assertEqual(self._run_test('testError', 1), 'E')
+
+    def testLongOutput(self):
+        classname = f'{__name__}.{self.Test.__qualname__}'
+        self.assertEqual(self._run_test('testSuccess', 2),
+                         f'testSuccess ({classname}) ... ok\n')
+        self.assertEqual(self._run_test('testSkip', 2),
+                         f"testSkip ({classname}) ... skipped 'skip'\n")
+        self.assertEqual(self._run_test('testFail', 2),
+                         f'testFail ({classname}) ... FAIL\n')
+        self.assertEqual(self._run_test('testError', 2),
+                         f'testError ({classname}) ... ERROR\n')
+
+    def testDotsOutputSubTestSuccess(self):
+        self.assertEqual(self._run_test('testSubTestSuccess', 1), '.')
+
+    def testLongOutputSubTestSuccess(self):
+        classname = f'{__name__}.{self.Test.__qualname__}'
+        self.assertEqual(self._run_test('testSubTestSuccess', 2),
+                         f'testSubTestSuccess ({classname}) ... ok\n')
+
+    def testDotsOutputSubTestMixed(self):
+        self.assertEqual(self._run_test('testSubTestMixed', 1), 'sFE')
+
+    def testLongOutputSubTestMixed(self):
+        classname = f'{__name__}.{self.Test.__qualname__}'
+        self.assertEqual(self._run_test('testSubTestMixed', 2),
+                f'testSubTestMixed ({classname}) ... \n'
+                f"  testSubTestMixed ({classname}) [skip] (b=2) ... skipped 'skip'\n"
+                f'  testSubTestMixed ({classname}) [fail] (c=3) ... FAIL\n'
+                f'  testSubTestMixed ({classname}) [error] (d=4) ... ERROR\n')
+
+    def testDotsOutputTearDownFail(self):
+        out = self._run_test('testSuccess', 1, AssertionError('fail'))
+        self.assertEqual(out, 'F')
+        out = self._run_test('testError', 1, AssertionError('fail'))
+        self.assertEqual(out, 'EF')
+        out = self._run_test('testFail', 1, Exception('error'))
+        self.assertEqual(out, 'FE')
+        out = self._run_test('testSkip', 1, AssertionError('fail'))
+        self.assertEqual(out, 'sF')
+
+    def testLongOutputTearDownFail(self):
+        classname = f'{__name__}.{self.Test.__qualname__}'
+        out = self._run_test('testSuccess', 2, AssertionError('fail'))
+        self.assertEqual(out,
+                         f'testSuccess ({classname}) ... FAIL\n')
+        out = self._run_test('testError', 2, AssertionError('fail'))
+        self.assertEqual(out,
+                         f'testError ({classname}) ... ERROR\n'
+                         f'testError ({classname}) ... FAIL\n')
+        out = self._run_test('testFail', 2, Exception('error'))
+        self.assertEqual(out,
+                         f'testFail ({classname}) ... FAIL\n'
+                         f'testFail ({classname}) ... ERROR\n')
+        out = self._run_test('testSkip', 2, AssertionError('fail'))
+        self.assertEqual(out,
+                         f"testSkip ({classname}) ... skipped 'skip'\n"
+                         f'testSkip ({classname}) ... FAIL\n')
 
 
 classDict = dict(unittest.TestResult.__dict__)

Misc/NEWS.d/next/Library/2021-09-05-13-15-08.bpo-25894.zjbi2f.rst

@@ -0,0 +1,4 @@
+:mod:`unittest` now always reports skipped and failed subtests separately:
+separate characters in default mode and separate lines in verbose mode. Also
+the test description is now output for errors in test method, class and
+module cleanups.