[3.5] bpo-29537: Tolerate legacy invalid bytecode (#169)

bpo-27286 fixed a problem where BUILD_MAP_UNPACK_WITH_CALL could
be emitted with an incorrect oparg value, causing the eval loop
to access the wrong stack entry when attempting to read the
function name.

The associated magic number change caused significant problems when
attempting to upgrade to 3.5.3 for anyone that relies on pre-cached
bytecode remaining valid across maintenance releases.

This patch restores the ability to import legacy bytecode generated
by 3.5.0, 3.5.1 or 3.5.2, and modifies the eval loop to
avoid any harmful consequences from the potentially malformed legacy
bytecode.

Original import patch by Petr Viktorin, eval loop patch by Serhiy Storchaka,
and tests and integration by Nick Coghlan.

3 files changed, 108 insertions, 4 deletions

diff --git a/Lib/test/test_extcall.py b/Lib/test/test_extcall.py
index 9cb0d38..94501de 100644
--- a/Lib/test/test_extcall.py
+++ b/Lib/test/test_extcall.py
@@ -52,15 +52,15 @@ Here we add keyword arguments
     >>> f(1, 2, **{'a': -1, 'b': 5}, **{'a': 4, 'c': 6})
     Traceback (most recent call last):
         ...
-    TypeError: f() got multiple values for keyword argument 'a'
+    TypeError: function got multiple values for keyword argument 'a'
     >>> f(1, 2, **{'a': -1, 'b': 5}, a=4, c=6)
     Traceback (most recent call last):
         ...
-    TypeError: f() got multiple values for keyword argument 'a'
+    TypeError: function got multiple values for keyword argument 'a'
     >>> f(1, 2, a=3, **{'a': 4}, **{'a': 5})
     Traceback (most recent call last):
         ...
-    TypeError: f() got multiple values for keyword argument 'a'
+    TypeError: function got multiple values for keyword argument 'a'
     >>> f(1, 2, 3, *[4, 5], **{'a':6, 'b':7})
     (1, 2, 3, 4, 5) {'a': 6, 'b': 7}
     >>> f(1, 2, 3, x=4, y=5, *(6, 7), **{'a':8, 'b': 9})
diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py
index 73f4c62..6d5a886 100644
--- a/Lib/test/test_importlib/source/test_file_loader.py
+++ b/Lib/test/test_importlib/source/test_file_loader.py
@@ -603,5 +603,109 @@ class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest,
                     util=importlib_util)
 
 
+###########################################################################
+# Issue #29537: Test backwards compatibility with legacy 3.5.0/1/2 bytecode
+###########################################################################
+
+class LegacyBytecodeTest:
+
+    def _test_legacy_magic(self, test, *, del_source=False):
+        # Replace the default magic number with one copied from a pyc file
+        # generated by Python 3.5.2
+        with util.create_modules('_temp') as mapping:
+            bc_path = self.manipulate_bytecode('_temp', mapping,
+                                    lambda bc: b'\x16\r\r\n' + bc[4:])
+            test('_temp', mapping, bc_path)
+
+LegacyBytecodeTestPEP451 = BadBytecodeTestPEP451
+LegacyBytecodeTestPEP302 = BadBytecodeTestPEP302
+
+# SourceLoader via both PEP 451 and 302 hooks
+
+class SourceLoaderLegacyBytecodeTest(LegacyBytecodeTest):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.loader = cls.machinery.SourceFileLoader
+
+    @util.writes_bytecode_files
+    def test_legacy_magic(self):
+        # The magic number from 3.5.0/1/2 should be accepted as is
+        def test(name, mapping, bytecode_path):
+            self.import_(mapping[name], name)
+            with open(bytecode_path, 'rb') as bytecode_file:
+                self.assertEqual(bytecode_file.read(4),
+                                 self.util._BACKCOMPAT_MAGIC_NUMBER)
+
+        self._test_legacy_magic(test)
+
+
+class SourceLoaderLegacyBytecodeTestPEP451(
+        SourceLoaderLegacyBytecodeTest, LegacyBytecodeTestPEP451):
+    pass
+
+
+(Frozen_SourceLegacyBytecodePEP451,
+ Source_SourceLegacyBytecodePEP451
+ ) = util.test_both(SourceLoaderLegacyBytecodeTestPEP451, importlib=importlib,
+                    machinery=machinery, abc=importlib_abc,
+                    util=importlib_util)
+
+
+class SourceLoaderLegacyBytecodeTestPEP302(
+        SourceLoaderLegacyBytecodeTest, LegacyBytecodeTestPEP302):
+    pass
+
+
+(Frozen_SourceLegacyBytecodePEP302,
+ Source_SourceLegacyBytecodePEP302
+ ) = util.test_both(SourceLoaderLegacyBytecodeTestPEP302, importlib=importlib,
+                    machinery=machinery, abc=importlib_abc,
+                    util=importlib_util)
+
+# SourcelessLoader via both PEP 451 and 302 hooks
+
+class SourcelessLoaderLegacyBytecodeTest(LegacyBytecodeTest):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.loader = cls.machinery.SourcelessFileLoader
+
+
+    @util.writes_bytecode_files
+    def test_legacy_magic(self):
+        # The magic number from 3.5.0/1/2 should be accepted as is
+        def test(name, mapping, bytecode_path):
+            self.import_(bytecode_path, name)
+            with open(bytecode_path, 'rb') as bytecode_file:
+                self.assertEqual(bytecode_file.read(4),
+                                 self.util._BACKCOMPAT_MAGIC_NUMBER)
+
+        self._test_legacy_magic(test)
+
+class SourcelessLoaderLegacyBytecodeTestPEP451(
+        SourcelessLoaderLegacyBytecodeTest, LegacyBytecodeTestPEP451):
+    pass
+
+(Frozen_SourcelessLegacyBytecodePEP451,
+ Source_SourcelessLegacyBytecodePEP451
+ ) = util.test_both(SourcelessLoaderLegacyBytecodeTestPEP451, importlib=importlib,
+                    machinery=machinery, abc=importlib_abc,
+                    util=importlib_util)
+
+
+class SourcelessLoaderLegacyBytecodeTestPEP302(SourcelessLoaderLegacyBytecodeTest,
+        LegacyBytecodeTestPEP302):
+    pass
+
+
+(Frozen_SourcelessLegacyBytecodePEP302,
+ Source_SourcelessLegacyBytecodePEP302
+ ) = util.test_both(SourcelessLoaderLegacyBytecodeTestPEP302, importlib=importlib,
+                    machinery=machinery, abc=importlib_abc,
+                    util=importlib_util)
+
+# End of Issue #29537 legacy bytecode compatibility tests
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/Lib/test/test_unpack_ex.py b/Lib/test/test_unpack_ex.py
index 74346b4..43cf638 100644
--- a/Lib/test/test_unpack_ex.py
+++ b/Lib/test/test_unpack_ex.py
@@ -251,7 +251,7 @@ Overridden parameters
     >>> f(x=5, **{'x': 3}, **{'x': 2})
     Traceback (most recent call last):
       ...
-    TypeError: f() got multiple values for keyword argument 'x'
+    TypeError: function got multiple values for keyword argument 'x'
 
     >>> f(**{1: 3}, **{1: 5})
     Traceback (most recent call last):