Issue #24164: Objects that need calling ``__new__`` with keyword arguments,

can now be pickled using pickle protocols older than protocol version 4.

6 files changed, 97 insertions, 32 deletions

diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst
index 50d2a6a..b735e9c 100644
--- a/Doc/whatsnew/3.6.rst
+++ b/Doc/whatsnew/3.6.rst
@@ -112,6 +112,15 @@ indexers. For example: ``subscript[0:10:2] == slice(0, 10, 2)``
 (Contributed by Joe Jevnik in :issue:`24379`.)
 
 
+pickle
+------
+
+Objects that need calling ``__new__`` with keyword arguments, can now be pickled
+using :ref:`pickle protocols <pickle-protocols>` older than protocol version 4.
+Protocol version 4 already supports this case.  (Contributed by Serhiy
+Storchaka in :issue:`24164`.)
+
+
 rlcomplete
 ----------
 
diff --git a/Lib/pickle.py b/Lib/pickle.py
index e93057a..d41753d 100644
--- a/Lib/pickle.py
+++ b/Lib/pickle.py
@@ -27,6 +27,7 @@ from types import FunctionType
 from copyreg import dispatch_table
 from copyreg import _extension_registry, _inverted_registry, _extension_cache
 from itertools import islice
+from functools import partial
 import sys
 from sys import maxsize
 from struct import pack, unpack
@@ -544,7 +545,7 @@ class _Pickler:
         write = self.write
 
         func_name = getattr(func, "__name__", "")
-        if self.proto >= 4 and func_name == "__newobj_ex__":
+        if self.proto >= 2 and func_name == "__newobj_ex__":
             cls, args, kwargs = args
             if not hasattr(cls, "__new__"):
                 raise PicklingError("args[0] from {} args has no __new__"
@@ -552,10 +553,16 @@ class _Pickler:
             if obj is not None and cls is not obj.__class__:
                 raise PicklingError("args[0] from {} args has the wrong class"
                                     .format(func_name))
-            save(cls)
-            save(args)
-            save(kwargs)
-            write(NEWOBJ_EX)
+            if self.proto >= 4:
+                save(cls)
+                save(args)
+                save(kwargs)
+                write(NEWOBJ_EX)
+            else:
+                func = partial(cls.__new__, cls, *args, **kwargs)
+                save(func)
+                save(())
+                write(REDUCE)
         elif self.proto >= 2 and func_name == "__newobj__":
             # A __reduce__ implementation can direct protocol 2 or newer to
             # use the more efficient NEWOBJ opcode, while still
diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
index 2ef48e6..fd641e3 100644
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -1580,16 +1580,14 @@ class AbstractPickleTests(unittest.TestCase):
         x.abc = 666
         for proto in protocols:
             with self.subTest(proto=proto):
-                if 2 <= proto < 4:
-                    self.assertRaises(ValueError, self.dumps, x, proto)
-                    continue
                 s = self.dumps(x, proto)
                 if proto < 1:
                     self.assertIn(b'\nL64206', s)  # LONG
                 elif proto < 2:
                     self.assertIn(b'M\xce\xfa', s)  # BININT2
+                elif proto < 4:
+                    self.assertIn(b'X\x04\x00\x00\x00FACE', s)  # BINUNICODE
                 else:
-                    assert proto >= 4
                     self.assertIn(b'\x8c\x04FACE', s)  # SHORT_BINUNICODE
                 self.assertFalse(opcode_in_pickle(pickle.NEWOBJ, s))
                 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ_EX, s),
diff --git a/Misc/NEWS b/Misc/NEWS
index cb02aae..802f4b6 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -51,6 +51,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #24164: Objects that need calling ``__new__`` with keyword arguments,
+  can now be pickled using pickle protocols older than protocol version 4.
+
 - Issue #25364: zipfile now works in threads disabled builds.
 
 - Issue #25328: smtpd's SMTPChannel now correctly raises a ValueError if both
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
index f3b73f1..abaf4e5 100644
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -153,6 +153,9 @@ typedef struct {
     PyObject *codecs_encode;
     /* builtins.getattr, used for saving nested names with protocol < 4 */
     PyObject *getattr;
+    /* functools.partial, used for implementing __newobj_ex__ with protocols
+       2 and 3 */
+    PyObject *partial;
 } PickleState;
 
 /* Forward declaration of the _pickle module definition. */
@@ -200,6 +203,7 @@ _Pickle_InitState(PickleState *st)
     PyObject *copyreg = NULL;
     PyObject *compat_pickle = NULL;
     PyObject *codecs = NULL;
+    PyObject *functools = NULL;
 
     builtins = PyEval_GetBuiltins();
     if (builtins == NULL)
@@ -314,12 +318,21 @@ _Pickle_InitState(PickleState *st)
     }
     Py_CLEAR(codecs);
 
+    functools = PyImport_ImportModule("functools");
+    if (!functools)
+        goto error;
+    st->partial = PyObject_GetAttrString(functools, "partial");
+    if (!st->partial)
+        goto error;
+    Py_CLEAR(functools);
+
     return 0;
 
   error:
     Py_CLEAR(copyreg);
     Py_CLEAR(compat_pickle);
     Py_CLEAR(codecs);
+    Py_CLEAR(functools);
     _Pickle_ClearState(st);
     return -1;
 }
@@ -3533,11 +3546,9 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj)
             PyErr_Clear();
         }
         else if (PyUnicode_Check(name)) {
-            if (self->proto >= 4) {
-                _Py_IDENTIFIER(__newobj_ex__);
-                use_newobj_ex = PyUnicode_Compare(
-                        name, _PyUnicode_FromId(&PyId___newobj_ex__)) == 0;
-            }
+            _Py_IDENTIFIER(__newobj_ex__);
+            use_newobj_ex = PyUnicode_Compare(
+                    name, _PyUnicode_FromId(&PyId___newobj_ex__)) == 0;
             if (!use_newobj_ex) {
                 _Py_IDENTIFIER(__newobj__);
                 use_newobj = PyUnicode_Compare(
@@ -3581,11 +3592,58 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj)
             return -1;
         }
 
-        if (save(self, cls, 0) < 0 ||
-            save(self, args, 0) < 0 ||
-            save(self, kwargs, 0) < 0 ||
-            _Pickler_Write(self, &newobj_ex_op, 1) < 0) {
-            return -1;
+        if (self->proto >= 4) {
+            if (save(self, cls, 0) < 0 ||
+                save(self, args, 0) < 0 ||
+                save(self, kwargs, 0) < 0 ||
+                _Pickler_Write(self, &newobj_ex_op, 1) < 0) {
+                return -1;
+            }
+        }
+        else {
+            PyObject *newargs;
+            PyObject *cls_new;
+            Py_ssize_t i;
+            _Py_IDENTIFIER(__new__);
+
+            newargs = PyTuple_New(Py_SIZE(args) + 2);
+            if (newargs == NULL)
+                return -1;
+
+            cls_new = _PyObject_GetAttrId(cls, &PyId___new__);
+            if (cls_new == NULL) {
+                Py_DECREF(newargs);
+                return -1;
+            }
+            PyTuple_SET_ITEM(newargs, 0, cls_new);
+            Py_INCREF(cls);
+            PyTuple_SET_ITEM(newargs, 1, cls);
+            for (i = 0; i < Py_SIZE(args); i++) {
+                PyObject *item = PyTuple_GET_ITEM(args, i);
+                Py_INCREF(item);
+                PyTuple_SET_ITEM(newargs, i + 2, item);
+            }
+
+            callable = PyObject_Call(st->partial, newargs, kwargs);
+            Py_DECREF(newargs);
+            if (callable == NULL)
+                return -1;
+
+            newargs = PyTuple_New(0);
+            if (newargs == NULL) {
+                Py_DECREF(callable);
+                return -1;
+            }
+
+            if (save(self, callable, 0) < 0 ||
+                save(self, newargs, 0) < 0 ||
+                _Pickler_Write(self, &reduce_op, 1) < 0) {
+                Py_DECREF(newargs);
+                Py_DECREF(callable);
+                return -1;
+            }
+            Py_DECREF(newargs);
+            Py_DECREF(callable);
         }
     }
     else if (use_newobj) {
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 4b091f5..bf0d30c 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -4101,7 +4101,7 @@ _PyObject_GetItemsIter(PyObject *obj, PyObject **listitems,
 }
 
 static PyObject *
-reduce_newobj(PyObject *obj, int proto)
+reduce_newobj(PyObject *obj)
 {
     PyObject *args = NULL, *kwargs = NULL;
     PyObject *copyreg;
@@ -4153,7 +4153,7 @@ reduce_newobj(PyObject *obj, int proto)
         }
         Py_DECREF(args);
     }
-    else if (proto >= 4) {
+    else {
         _Py_IDENTIFIER(__newobj_ex__);
 
         newobj = _PyObject_GetAttrId(copyreg, &PyId___newobj_ex__);
@@ -4171,16 +4171,6 @@ reduce_newobj(PyObject *obj, int proto)
             return NULL;
         }
     }
-    else {
-        PyErr_SetString(PyExc_ValueError,
-                        "must use protocol 4 or greater to copy this "
-                        "object; since __getnewargs_ex__ returned "
-                        "keyword arguments.");
-        Py_DECREF(args);
-        Py_DECREF(kwargs);
-        Py_DECREF(copyreg);
-        return NULL;
-    }
 
     state = _PyObject_GetState(obj);
     if (state == NULL) {
@@ -4225,7 +4215,7 @@ _common_reduce(PyObject *self, int proto)
     PyObject *copyreg, *res;
 
     if (proto >= 2)
-        return reduce_newobj(self, proto);
+        return reduce_newobj(self);
 
     copyreg = import_copyreg();
     if (!copyreg)