[3.5] bpo-29537: Tolerate legacy invalid bytecode (#169)

bpo-27286 fixed a problem where BUILD_MAP_UNPACK_WITH_CALL could
be emitted with an incorrect oparg value, causing the eval loop
to access the wrong stack entry when attempting to read the
function name.

The associated magic number change caused significant problems when
attempting to upgrade to 3.5.3 for anyone that relies on pre-cached
bytecode remaining valid across maintenance releases.

This patch restores the ability to import legacy bytecode generated
by 3.5.0, 3.5.1 or 3.5.2, and modifies the eval loop to
avoid any harmful consequences from the potentially malformed legacy
bytecode.

Original import patch by Petr Viktorin, eval loop patch by Serhiy Storchaka,
and tests and integration by Nick Coghlan.

1 files changed, 17 insertions, 2 deletions

diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index 90cb2de..dd41a3a 100644
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -259,6 +259,21 @@ PyRun_InteractiveOneFlags(FILE *fp, const char *filename_str, PyCompilerFlags *f
 }
 
 
+/* Issue #29537: handle issue27286 bytecode incompatibility
+ *   See Lib/importlib/_bootstrap_external.py for general discussion
+ */
+extern PY_UINT32_T _Py_BACKCOMPAT_HALF_MAGIC;
+static int
+_check_half_magic(unsigned int read_value, unsigned int halfmagic) {
+    return (read_value == halfmagic || read_value == _Py_BACKCOMPAT_HALF_MAGIC);
+}
+
+extern PY_UINT32_T _Py_BACKCOMPAT_MAGIC_NUMBER;
+static int
+_check_magic(long read_value, long magic) {
+    return (read_value == magic || read_value == _Py_BACKCOMPAT_MAGIC_NUMBER);
+}
+
 /* Check whether a file maybe a pyc file: Look at the extension,
    the file type, and, if we may close it, at the first few bytes. */
 
@@ -290,7 +305,7 @@ maybe_pyc_file(FILE *fp, const char* filename, const char* ext, int closeit)
         int ispyc = 0;
         if (ftell(fp) == 0) {
             if (fread(buf, 1, 2, fp) == 2 &&
-                ((unsigned int)buf[1]<<8 | buf[0]) == halfmagic)
+                _check_half_magic(((unsigned int)buf[1]<<8 | buf[0]), halfmagic))
                 ispyc = 1;
             rewind(fp);
         }
@@ -988,7 +1003,7 @@ run_pyc_file(FILE *fp, const char *filename, PyObject *globals,
     long PyImport_GetMagicNumber(void);
 
     magic = PyMarshal_ReadLongFromFile(fp);
-    if (magic != PyImport_GetMagicNumber()) {
+    if (!_check_magic(magic, PyImport_GetMagicNumber())) {
         if (!PyErr_Occurred())
             PyErr_SetString(PyExc_RuntimeError,
                        "Bad magic number in .pyc file");