unittest runner: Exit code 5 if no tests were run

As discussed in https://discuss.python.org/t/unittest-fail-if-zero-tests-were-discovered/21498/7

It is common for test runner misconfiguration to fail to find any tests,
this should be an error.

Fixes: python#62432

Author: stefanor

Lib/test/test_unittest/test_program.py

@@ -71,6 +71,9 @@ def testExpectedFailure(self):
         def testUnexpectedSuccess(self):
             pass
 
+    class Empty(unittest.TestCase):
+        pass
+
     class TestLoader(unittest.TestLoader):
         """Test loader that returns a suite containing testsuite."""
 
@@ -134,12 +137,13 @@ def test_NonExit(self):
 
     def test_Exit(self):
         stream = BufferedWriter()
-        with self.assertRaises(SystemExit):
+        with self.assertRaises(SystemExit) as cm:
             unittest.main(
                 argv=["foobar"],
                 testRunner=unittest.TextTestRunner(stream=stream),
                 exit=True,
                 testLoader=self.TestLoader(self.FooBar))
+        self.assertEqual(cm.exception.code, 1)
         out = stream.getvalue()
         self.assertIn('\nFAIL: testFail ', out)
         self.assertIn('\nERROR: testError ', out)
@@ -163,6 +167,17 @@ def test_ExitAsDefault(self):
                     'expected failures=1, unexpected successes=1)\n')
         self.assertTrue(out.endswith(expected))
 
+    def test_ExitEmptySuite(self):
+        stream = BufferedWriter()
+        with self.assertRaises(SystemExit) as cm:
+            unittest.main(
+                argv=["empty"],
+                testRunner=unittest.TextTestRunner(stream=stream),
+                testLoader=self.TestLoader(self.Empty))
+        self.assertEqual(cm.exception.code, 5)
+        out = stream.getvalue()
+        self.assertIn('\nNO TESTS RUN\n', out)
+
 
 class InitialisableProgram(unittest.TestProgram):
     exit = False

Lib/unittest/main.py

@@ -273,6 +273,11 @@ def runTests(self):
             testRunner = self.testRunner
         self.result = testRunner.run(self.test)
         if self.exit:
-            sys.exit(not self.result.wasSuccessful())
+            if self.result.testsRun == 0:
+                sys.exit(5)
+            elif not self.result.wasSuccessful():
+                sys.exit(1)
+            else:
+                sys.exit(0)
 
 main = TestProgram

Misc/NEWS.d/next/Library/2023-02-19-12-37-08.gh-issue-62432.GnBFIB.rst

@@ -0,0 +1,3 @@
+The :mod:`unittest` runner will now exit with status code 5, if no tests
+were run. It is common for test runner misconfiguration to fail to find any
+tests, this should be an error.