bpo-30764: regrtest: add --fail-env-changed option (#2402)

* bpo-30764: regrtest: change exit code on failure

* Exit code 2 if failed tests ("bad")
* Exit code 3 if interrupted

* bpo-30764: regrtest: add --fail-env-changed option

If the option is set, mark a test as failed if it alters the
environment, for example if it creates a file without removing it.

2 files changed, 12 insertions, 1 deletions

diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py
index bf64062..2315cd5 100644
--- a/Lib/test/libregrtest/cmdline.py
+++ b/Lib/test/libregrtest/cmdline.py
@@ -255,6 +255,9 @@ def _create_parser():
                             ' , don\'t execute them')
     group.add_argument('-P', '--pgo', dest='pgo', action='store_true',
                        help='enable Profile Guided Optimization training')
+    group.add_argument('--fail-env-changed', action='store_true',
+                       help='if a test file alters the environment, mark '
+                            'the test as failed')
 
     return parser
 
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index 1a77655..571eb61 100644
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -478,6 +478,8 @@ class Regrtest:
             result = "FAILURE"
         elif self.interrupted:
             result = "INTERRUPTED"
+        elif self.environment_changed and self.ns.fail_env_changed:
+            result = "ENV CHANGED"
         else:
             result = "SUCCESS"
         print("Tests result: %s" % result)
@@ -538,7 +540,13 @@ class Regrtest:
             self.rerun_failed_tests()
 
         self.finalize()
-        sys.exit(len(self.bad) > 0 or self.interrupted)
+        if self.bad:
+            sys.exit(2)
+        if self.interrupted:
+            sys.exit(130)
+        if self.ns.fail_env_changed and self.environment_changed:
+            sys.exit(3)
+        sys.exit(0)
 
 
 def removepy(names):