bpo-43950: Add option to opt-out of PEP-657 (GH-27023)

Co-authored-by: Pablo Galindo <Pablogsal@gmail.com>
Co-authored-by: Batuhan Taskaya <batuhanosmantaskaya@gmail.com>
Co-authored-by: Ammar Askar <ammar@ammaraskar.com>

11 files changed, 177 insertions, 28 deletions

diff --git a/Lib/idlelib/idle_test/test_run.py b/Lib/idlelib/idle_test/test_run.py
index b289fa7..d859ffc 100644
--- a/Lib/idlelib/idle_test/test_run.py
+++ b/Lib/idlelib/idle_test/test_run.py
@@ -3,7 +3,7 @@
 from idlelib import run
 import io
 import sys
-from test.support import captured_output, captured_stderr
+from test.support import captured_output, captured_stderr, has_no_debug_ranges
 import unittest
 from unittest import mock
 import idlelib
@@ -33,9 +33,14 @@ class ExceptionTest(unittest.TestCase):
                         run.print_exception()
 
         tb = output.getvalue().strip().splitlines()
-        self.assertEqual(13, len(tb))
-        self.assertIn('UnhashableException: ex2', tb[4])
-        self.assertIn('UnhashableException: ex1', tb[12])
+        if has_no_debug_ranges():
+            self.assertEqual(11, len(tb))
+            self.assertIn('UnhashableException: ex2', tb[3])
+            self.assertIn('UnhashableException: ex1', tb[10])
+        else:
+            self.assertEqual(13, len(tb))
+            self.assertIn('UnhashableException: ex2', tb[4])
+            self.assertIn('UnhashableException: ex1', tb[12])
 
     data = (('1/0', ZeroDivisionError, "division by zero\n"),
             ('abc', NameError, "name 'abc' is not defined. "
diff --git a/Lib/test/_test_embed_set_config.py b/Lib/test/_test_embed_set_config.py
index 82c5d82..23c927e 100644
--- a/Lib/test/_test_embed_set_config.py
+++ b/Lib/test/_test_embed_set_config.py
@@ -61,6 +61,7 @@ class SetConfigTests(unittest.TestCase):
             'faulthandler',
             'tracemalloc',
             'import_time',
+            'no_debug_ranges',
             'show_ref_count',
             'dump_refs',
             'malloc_stats',
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 933c2c9..59b8f44 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -415,6 +415,14 @@ def requires_lzma(reason='requires lzma'):
         lzma = None
     return unittest.skipUnless(lzma, reason)
 
+def has_no_debug_ranges():
+    import _testinternalcapi
+    config = _testinternalcapi.get_config()
+    return bool(config['no_debug_ranges'])
+
+def requires_debug_ranges(reason='requires co_positions / debug_ranges'):
+    return unittest.skipIf(has_no_debug_ranges(), reason)
+
 requires_legacy_unicode_capi = unittest.skipUnless(unicode_legacy_string,
                         'requires legacy Unicode C API')
 
diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py
index ccb8da6..988790b 100644
--- a/Lib/test/test_code.py
+++ b/Lib/test/test_code.py
@@ -137,7 +137,8 @@ try:
 except ImportError:
     ctypes = None
 from test.support import (run_doctest, run_unittest, cpython_only,
-                          check_impl_detail)
+                          check_impl_detail, requires_debug_ranges)
+from test.support.script_helper import assert_python_ok
 
 
 def consts(t):
@@ -325,6 +326,7 @@ class CodeTest(unittest.TestCase):
         new_code = code = func.__code__.replace(co_linetable=b'')
         self.assertEqual(list(new_code.co_lines()), [])
 
+    @requires_debug_ranges()
     def test_co_positions_artificial_instructions(self):
         import dis
 
@@ -372,8 +374,32 @@ class CodeTest(unittest.TestCase):
             ]
         )
 
+    def test_endline_and_columntable_none_when_no_debug_ranges(self):
+        # Make sure that if `-X no_debug_ranges` is used, the endlinetable and
+        # columntable are None.
+        code = textwrap.dedent("""
+            def f():
+                pass
+
+            assert f.__code__.co_endlinetable is None
+            assert f.__code__.co_columntable is None
+            """)
+        assert_python_ok('-X', 'no_debug_ranges', '-c', code, __cleanenv=True)
+
+    def test_endline_and_columntable_none_when_no_debug_ranges_env(self):
+        # Same as above but using the environment variable opt out.
+        code = textwrap.dedent("""
+            def f():
+                pass
+
+            assert f.__code__.co_endlinetable is None
+            assert f.__code__.co_columntable is None
+            """)
+        assert_python_ok('-c', code, PYTHONNODEBUGRANGES='1', __cleanenv=True)
+
     # co_positions behavior when info is missing.
 
+    @requires_debug_ranges()
     def test_co_positions_empty_linetable(self):
         def func():
             x = 1
@@ -382,6 +408,7 @@ class CodeTest(unittest.TestCase):
             self.assertIsNone(line)
             self.assertEqual(end_line, new_code.co_firstlineno + 1)
 
+    @requires_debug_ranges()
     def test_co_positions_empty_endlinetable(self):
         def func():
             x = 1
@@ -390,6 +417,7 @@ class CodeTest(unittest.TestCase):
             self.assertEqual(line, new_code.co_firstlineno + 1)
             self.assertIsNone(end_line)
 
+    @requires_debug_ranges()
     def test_co_positions_empty_columntable(self):
         def func():
             x = 1
diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py
index bc8c57d..c994741 100644
--- a/Lib/test/test_compile.py
+++ b/Lib/test/test_compile.py
@@ -9,7 +9,7 @@ import tempfile
 import types
 import textwrap
 from test import support
-from test.support import script_helper
+from test.support import script_helper, requires_debug_ranges
 from test.support.os_helper import FakePath
 
 
@@ -985,7 +985,7 @@ if 1:
             elif instr.opname in HANDLED_JUMPS:
                 self.assertNotEqual(instr.arg, (line + 1)*INSTR_SIZE)
 
-
+@requires_debug_ranges()
 class TestSourcePositions(unittest.TestCase):
     # Ensure that compiled code snippets have correct line and column numbers
     # in `co_positions()`.
diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py
index 54a123e..d1f1eee 100644
--- a/Lib/test/test_dis.py
+++ b/Lib/test/test_dis.py
@@ -1,6 +1,6 @@
 # Minimal tests for dis module
 
-from test.support import captured_stdout
+from test.support import captured_stdout, requires_debug_ranges
 from test.support.bytecode_helper import BytecodeTestCase
 import unittest
 import sys
@@ -1192,6 +1192,7 @@ class InstructionTests(InstructionTestCase):
         actual = dis.get_instructions(jumpy, first_line=expected_jumpy_line)
         self.assertInstructionsEqual(list(actual), expected_opinfo_jumpy)
 
+    @requires_debug_ranges()
     def test_co_positions(self):
         code = compile('f(\n  x, y, z\n)', '<test>', 'exec')
         positions = [
@@ -1210,6 +1211,7 @@ class InstructionTests(InstructionTestCase):
         ]
         self.assertEqual(positions, expected)
 
+    @requires_debug_ranges()
     def test_co_positions_missing_info(self):
         code = compile('x, y, z', '<test>', 'exec')
         code_without_column_table = code.replace(co_columntable=b'')
diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py
index 06d9d5d..642188f 100644
--- a/Lib/test/test_doctest.py
+++ b/Lib/test/test_doctest.py
@@ -2808,10 +2808,12 @@ out of the binary module.
 
 try:
     os.fsencode("foo-bär@baz.py")
+    supports_unicode = True
 except UnicodeEncodeError:
     # Skip the test: the filesystem encoding is unable to encode the filename
-    pass
-else:
+    supports_unicode = False
+
+if supports_unicode and not support.has_no_debug_ranges():
     def test_unicode(): """
 Check doctest with a non-ascii filename:
 
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index c50defd..8e3dd50 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -369,6 +369,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'faulthandler': 0,
         'tracemalloc': 0,
         'import_time': 0,
+        'no_debug_ranges': 0,
         'show_ref_count': 0,
         'dump_refs': 0,
         'malloc_stats': 0,
@@ -798,6 +799,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             'hash_seed': 123,
             'tracemalloc': 2,
             'import_time': 1,
+            'no_debug_ranges': 1,
             'show_ref_count': 1,
             'malloc_stats': 1,
 
@@ -858,6 +860,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             'hash_seed': 42,
             'tracemalloc': 2,
             'import_time': 1,
+            'no_debug_ranges': 1,
             'malloc_stats': 1,
             'inspect': 1,
             'optimization_level': 2,
@@ -887,6 +890,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             'hash_seed': 42,
             'tracemalloc': 2,
             'import_time': 1,
+            'no_debug_ranges': 1,
             'malloc_stats': 1,
             'inspect': 1,
             'optimization_level': 2,
diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py
index 7bcf8e8..152301f 100644
--- a/Lib/test/test_marshal.py
+++ b/Lib/test/test_marshal.py
@@ -1,5 +1,6 @@
 from test import support
-from test.support import os_helper
+from test.support import os_helper, requires_debug_ranges
+from test.support.script_helper import assert_python_ok
 import array
 import io
 import marshal
@@ -7,6 +8,7 @@ import sys
 import unittest
 import os
 import types
+import textwrap
 
 try:
     import _testcapi
@@ -126,6 +128,31 @@ class CodeTestCase(unittest.TestCase):
         self.assertEqual(co1.co_filename, "f1")
         self.assertEqual(co2.co_filename, "f2")
 
+    @requires_debug_ranges()
+    def test_no_columntable_and_endlinetable_with_no_debug_ranges(self):
+        # Make sure when demarshalling objects with `-X no_debug_ranges`
+        # that the columntable and endlinetable are None.
+        co = ExceptionTestCase.test_exceptions.__code__
+        code = textwrap.dedent("""
+        import sys
+        import marshal
+        with open(sys.argv[1], 'rb') as f:
+            co = marshal.load(f)
+
+            assert co.co_endlinetable is None
+            assert co.co_columntable is None
+        """)
+
+        try:
+            with open(os_helper.TESTFN, 'wb') as f:
+                marshal.dump(co, f)
+
+            assert_python_ok('-X', 'no_debug_ranges',
+                             '-c', code, os_helper.TESTFN,
+                             __cleanenv=True)
+        finally:
+            os_helper.unlink(os_helper.TESTFN)
+
     @support.cpython_only
     def test_same_filename_used(self):
         s = """def f(): pass\ndef g(): pass"""
diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py
index 54f592a..610ae3f 100644
--- a/Lib/test/test_traceback.py
+++ b/Lib/test/test_traceback.py
@@ -8,9 +8,10 @@ import inspect
 import unittest
 import re
 from test import support
-from test.support import Error, captured_output, cpython_only, ALWAYS_EQ
+from test.support import (Error, captured_output, cpython_only, ALWAYS_EQ,
+                          requires_debug_ranges, has_no_debug_ranges)
 from test.support.os_helper import TESTFN, unlink
-from test.support.script_helper import assert_python_ok
+from test.support.script_helper import assert_python_ok, assert_python_failure
 import textwrap
 
 import traceback
@@ -75,6 +76,49 @@ class TracebackCases(unittest.TestCase):
         self.assertEqual(len(err), 3)
         self.assertEqual(err[1].strip(), "bad syntax")
 
+    def test_no_caret_with_no_debug_ranges_flag(self):
+        # Make sure that if `-X no_debug_ranges` is used, there are no carets
+        # in the traceback.
+        try:
+            with open(TESTFN, 'w') as f:
+                f.write("x = 1 / 0\n")
+
+            _, _, stderr = assert_python_failure(
+                '-X', 'no_debug_ranges', TESTFN, __cleanenv=True)
+
+            lines = stderr.splitlines()
+            self.assertEqual(len(lines), 4)
+            self.assertEqual(lines[0], b'Traceback (most recent call last):')
+            self.assertIn(b'line 1, in <module>', lines[1])
+            self.assertEqual(lines[2], b'    x = 1 / 0')
+            self.assertEqual(lines[3], b'ZeroDivisionError: division by zero')
+        finally:
+            unlink(TESTFN)
+
+    def test_no_caret_with_no_debug_ranges_flag_python_traceback(self):
+        code = textwrap.dedent("""
+            import traceback
+            try:
+                x = 1 / 0
+            except:
+                traceback.print_exc()
+            """)
+        try:
+            with open(TESTFN, 'w') as f:
+                f.write(code)
+
+            _, _, stderr = assert_python_ok(
+                '-X', 'no_debug_ranges', TESTFN, __cleanenv=True)
+
+            lines = stderr.splitlines()
+            self.assertEqual(len(lines), 4)
+            self.assertEqual(lines[0], b'Traceback (most recent call last):')
+            self.assertIn(b'line 4, in <module>', lines[1])
+            self.assertEqual(lines[2], b'    x = 1 / 0')
+            self.assertEqual(lines[3], b'ZeroDivisionError: division by zero')
+        finally:
+            unlink(TESTFN)
+
     def test_bad_indentation(self):
         err = self.get_exception_format(self.syntax_error_bad_indentation,
                                         IndentationError)
@@ -155,9 +199,10 @@ class TracebackCases(unittest.TestCase):
             self.assertTrue(stdout[2].endswith(err_line),
                 "Invalid traceback line: {0!r} instead of {1!r}".format(
                     stdout[2], err_line))
-            self.assertTrue(stdout[4] == err_msg,
+            actual_err_msg = stdout[3 if has_no_debug_ranges() else 4]
+            self.assertTrue(actual_err_msg == err_msg,
                 "Invalid error message: {0!r} instead of {1!r}".format(
-                    stdout[4], err_msg))
+                    actual_err_msg, err_msg))
 
         do_test("", "foo", "ascii", 3)
         for charset in ("ascii", "iso-8859-1", "utf-8", "GBK"):
@@ -273,6 +318,7 @@ class TracebackCases(unittest.TestCase):
             '(exc, /, value=<implicit>)')
 
 
+@requires_debug_ranges()
 class TracebackErrorLocationCaretTests(unittest.TestCase):
     """
     Tests for printing code error expressions as part of PEP 657
@@ -362,6 +408,7 @@ class TracebackErrorLocationCaretTests(unittest.TestCase):
 
 
 @cpython_only
+@requires_debug_ranges()
 class CPythonTracebackErrorCaretTests(TracebackErrorLocationCaretTests):
     """
     Same set of tests as above but with Python's internal traceback printing.
@@ -424,9 +471,13 @@ class TracebackFormatTests(unittest.TestCase):
 
         # Make sure that the traceback is properly indented.
         tb_lines = python_fmt.splitlines()
-        self.assertEqual(len(tb_lines), 7)
         banner = tb_lines[0]
-        location, source_line = tb_lines[-3], tb_lines[-2]
+        if has_no_debug_ranges():
+            self.assertEqual(len(tb_lines), 5)
+            location, source_line = tb_lines[-2], tb_lines[-1]
+        else:
+            self.assertEqual(len(tb_lines), 7)
+            location, source_line = tb_lines[-3], tb_lines[-2]
         self.assertTrue(banner.startswith('Traceback'))
         self.assertTrue(location.startswith('  File'))
         self.assertTrue(source_line.startswith('    raise'))
@@ -668,10 +719,12 @@ class TracebackFormatTests(unittest.TestCase):
         actual = stderr_g.getvalue().splitlines()
         self.assertEqual(actual, expected)
 
+    @requires_debug_ranges()
     def test_recursive_traceback_python(self):
         self._check_recursive_traceback_display(traceback.print_exc)
 
     @cpython_only
+    @requires_debug_ranges()
     def test_recursive_traceback_cpython_internal(self):
         from _testcapi import exception_print
         def render_exc():
@@ -713,11 +766,16 @@ class TracebackFormatTests(unittest.TestCase):
             exception_print(exc_val)
 
         tb = stderr_f.getvalue().strip().splitlines()
-        self.assertEqual(13, len(tb))
-        self.assertEqual(context_message.strip(), tb[6])
-        self.assertIn('UnhashableException: ex2', tb[4])
-        self.assertIn('UnhashableException: ex1', tb[12])
-
+        if has_no_debug_ranges():
+            self.assertEqual(11, len(tb))
+            self.assertEqual(context_message.strip(), tb[5])
+            self.assertIn('UnhashableException: ex2', tb[3])
+            self.assertIn('UnhashableException: ex1', tb[10])
+        else:
+            self.assertEqual(13, len(tb))
+            self.assertEqual(context_message.strip(), tb[6])
+            self.assertIn('UnhashableException: ex2', tb[4])
+            self.assertIn('UnhashableException: ex1', tb[12])
 
 cause_message = (
     "\nThe above exception was the direct cause "
@@ -746,8 +804,12 @@ class BaseExceptionReportingTests:
 
     def check_zero_div(self, msg):
         lines = msg.splitlines()
-        self.assertTrue(lines[-4].startswith('  File'))
-        self.assertIn('1/0 # In zero_div', lines[-3])
+        if has_no_debug_ranges():
+            self.assertTrue(lines[-3].startswith('  File'))
+            self.assertIn('1/0 # In zero_div', lines[-2])
+        else:
+            self.assertTrue(lines[-4].startswith('  File'))
+            self.assertIn('1/0 # In zero_div', lines[-3])
         self.assertTrue(lines[-1].startswith('ZeroDivisionError'), lines[-1])
 
     def test_simple(self):
@@ -756,11 +818,15 @@ class BaseExceptionReportingTests:
         except ZeroDivisionError as _:
             e = _
         lines = self.get_report(e).splitlines()
-        self.assertEqual(len(lines), 5)
+        if has_no_debug_ranges():
+            self.assertEqual(len(lines), 4)
+            self.assertTrue(lines[3].startswith('ZeroDivisionError'))
+        else:
+            self.assertEqual(len(lines), 5)
+            self.assertTrue(lines[4].startswith('ZeroDivisionError'))
         self.assertTrue(lines[0].startswith('Traceback'))
         self.assertTrue(lines[1].startswith('  File'))
         self.assertIn('1/0 # Marker', lines[2])
-        self.assertTrue(lines[4].startswith('ZeroDivisionError'))
 
     def test_cause(self):
         def inner_raise():
@@ -799,11 +865,15 @@ class BaseExceptionReportingTests:
         except ZeroDivisionError as _:
             e = _
         lines = self.get_report(e).splitlines()
-        self.assertEqual(len(lines), 5)
+        if has_no_debug_ranges():
+            self.assertEqual(len(lines), 4)
+            self.assertTrue(lines[3].startswith('ZeroDivisionError'))
+        else:
+            self.assertEqual(len(lines), 5)
+            self.assertTrue(lines[4].startswith('ZeroDivisionError'))
         self.assertTrue(lines[0].startswith('Traceback'))
         self.assertTrue(lines[1].startswith('  File'))
         self.assertIn('ZeroDivisionError from None', lines[2])
-        self.assertTrue(lines[4].startswith('ZeroDivisionError'))
 
     def test_cause_and_context(self):
         # When both a cause and a context are set, only the cause should be
@@ -1527,6 +1597,7 @@ class TestTracebackException(unittest.TestCase):
         exc = traceback.TracebackException(Exception, Exception("haven"), None)
         self.assertEqual(list(exc.format()), ["Exception: haven\n"])
 
+    @requires_debug_ranges()
     def test_print(self):
         def f():
             x = 12
diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py
index 861ebe3..4e88902 100644
--- a/Lib/test/test_zipimport.py
+++ b/Lib/test/test_zipimport.py
@@ -718,6 +718,7 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
             print_tb(tb, 1, s)
             self.assertTrue(s.getvalue().endswith(
                 '    def do_raise(): raise TypeError\n'
+                '' if support.has_no_debug_ranges() else
                 '                    ^^^^^^^^^^^^^^^\n'
             ))
         else: