gh-62432: unittest runner: Exit code 5 if no tests were run (#102051)

As discussed in https://discuss.python.org/t/unittest-fail-if-zero-tests-were-discovered/21498/7

It is common for test runner misconfiguration to fail to find any tests,
This should be an error.

Fixes: #62432

3 files changed, 48 insertions, 20 deletions

diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py
index f138f68..f6d52f9 100644
--- a/Lib/test/test_unittest/test_program.py
+++ b/Lib/test/test_unittest/test_program.py
@@ -71,15 +71,22 @@ class Test_TestProgram(unittest.TestCase):
         def testUnexpectedSuccess(self):
             pass
 
-    class FooBarLoader(unittest.TestLoader):
-        """Test loader that returns a suite containing FooBar."""
+    class Empty(unittest.TestCase):
+        pass
+
+    class TestLoader(unittest.TestLoader):
+        """Test loader that returns a suite containing the supplied testcase."""
+
+        def __init__(self, testcase):
+            self.testcase = testcase
+
         def loadTestsFromModule(self, module):
             return self.suiteClass(
-                [self.loadTestsFromTestCase(Test_TestProgram.FooBar)])
+                [self.loadTestsFromTestCase(self.testcase)])
 
         def loadTestsFromNames(self, names, module):
             return self.suiteClass(
-                [self.loadTestsFromTestCase(Test_TestProgram.FooBar)])
+                [self.loadTestsFromTestCase(self.testcase)])
 
     def test_defaultTest_with_string(self):
         class FakeRunner(object):
@@ -92,7 +99,7 @@ class Test_TestProgram(unittest.TestCase):
         runner = FakeRunner()
         program = unittest.TestProgram(testRunner=runner, exit=False,
                                        defaultTest='test.test_unittest',
-                                       testLoader=self.FooBarLoader())
+                                       testLoader=self.TestLoader(self.FooBar))
         sys.argv = old_argv
         self.assertEqual(('test.test_unittest',), program.testNames)
 
@@ -108,7 +115,7 @@ class Test_TestProgram(unittest.TestCase):
         program = unittest.TestProgram(
             testRunner=runner, exit=False,
             defaultTest=['test.test_unittest', 'test.test_unittest2'],
-            testLoader=self.FooBarLoader())
+            testLoader=self.TestLoader(self.FooBar))
         sys.argv = old_argv
         self.assertEqual(['test.test_unittest', 'test.test_unittest2'],
                           program.testNames)
@@ -118,7 +125,7 @@ class Test_TestProgram(unittest.TestCase):
         program = unittest.main(exit=False,
                                 argv=["foobar"],
                                 testRunner=unittest.TextTestRunner(stream=stream),
-                                testLoader=self.FooBarLoader())
+                                testLoader=self.TestLoader(self.FooBar))
         self.assertTrue(hasattr(program, 'result'))
         out = stream.getvalue()
         self.assertIn('\nFAIL: testFail ', out)
@@ -130,13 +137,13 @@ class Test_TestProgram(unittest.TestCase):
 
     def test_Exit(self):
         stream = BufferedWriter()
-        self.assertRaises(
-            SystemExit,
-            unittest.main,
-            argv=["foobar"],
-            testRunner=unittest.TextTestRunner(stream=stream),
-            exit=True,
-            testLoader=self.FooBarLoader())
+        with self.assertRaises(SystemExit) as cm:
+            unittest.main(
+                argv=["foobar"],
+                testRunner=unittest.TextTestRunner(stream=stream),
+                exit=True,
+                testLoader=self.TestLoader(self.FooBar))
+        self.assertEqual(cm.exception.code, 1)
         out = stream.getvalue()
         self.assertIn('\nFAIL: testFail ', out)
         self.assertIn('\nERROR: testError ', out)
@@ -147,12 +154,11 @@ class Test_TestProgram(unittest.TestCase):
 
     def test_ExitAsDefault(self):
         stream = BufferedWriter()
-        self.assertRaises(
-            SystemExit,
-            unittest.main,
-            argv=["foobar"],
-            testRunner=unittest.TextTestRunner(stream=stream),
-            testLoader=self.FooBarLoader())
+        with self.assertRaises(SystemExit):
+            unittest.main(
+                argv=["foobar"],
+                testRunner=unittest.TextTestRunner(stream=stream),
+                testLoader=self.TestLoader(self.FooBar))
         out = stream.getvalue()
         self.assertIn('\nFAIL: testFail ', out)
         self.assertIn('\nERROR: testError ', out)
@@ -161,6 +167,17 @@ class Test_TestProgram(unittest.TestCase):
                     'expected failures=1, unexpected successes=1)\n')
         self.assertTrue(out.endswith(expected))
 
+    def test_ExitEmptySuite(self):
+        stream = BufferedWriter()
+        with self.assertRaises(SystemExit) as cm:
+            unittest.main(
+                argv=["empty"],
+                testRunner=unittest.TextTestRunner(stream=stream),
+                testLoader=self.TestLoader(self.Empty))
+        self.assertEqual(cm.exception.code, 5)
+        out = stream.getvalue()
+        self.assertIn('\nNO TESTS RAN\n', out)
+
 
 class InitialisableProgram(unittest.TestProgram):
     exit = False
diff --git a/Lib/test/test_unittest/test_result.py b/Lib/test/test_unittest/test_result.py
index 37d0fe1..db551b7 100644
--- a/Lib/test/test_unittest/test_result.py
+++ b/Lib/test/test_unittest/test_result.py
@@ -451,6 +451,7 @@ class Test_TestResult(unittest.TestCase):
         stream = BufferedWriter()
         runner = unittest.TextTestRunner(stream=stream, failfast=True)
         def test(result):
+            result.testsRun += 1
             self.assertTrue(result.failfast)
         result = runner.run(test)
         stream.flush()
diff --git a/Lib/test/test_unittest/test_runner.py b/Lib/test/test_unittest/test_runner.py
index ceb4c8a..f3b2c0c 100644
--- a/Lib/test/test_unittest/test_runner.py
+++ b/Lib/test/test_unittest/test_runner.py
@@ -577,6 +577,16 @@ class TestClassCleanup(unittest.TestCase):
                 'inner setup', 'inner test', 'inner cleanup',
                 'end outer test', 'outer cleanup'])
 
+    def test_run_empty_suite_error_message(self):
+        class EmptyTest(unittest.TestCase):
+            pass
+
+        suite = unittest.defaultTestLoader.loadTestsFromTestCase(EmptyTest)
+        runner = getRunner()
+        runner.run(suite)
+
+        self.assertIn("\nNO TESTS RAN\n", runner.stream.getvalue())
+
 
 class TestModuleCleanUp(unittest.TestCase):
     def test_add_and_do_ModuleCleanup(self):