From b8e54dd806e75f3591d8b7f07676a5738dad019d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 30 Dec 2015 20:43:29 +0200 Subject: Issue #22995: Instances of extension types with a state that aren't subclasses of list or dict and haven't implemented any pickle-related methods (__reduce__, __reduce_ex__, __getnewargs__, __getnewargs_ex__, or __getstate__), can no longer be pickled. Including memoryview. --- Lib/test/test_buffer.py | 13 +++++++++++++ Lib/test/test_csv.py | 13 +++++++++++++ Lib/test/test_memoryview.py | 15 +++++++++++++++ Misc/NEWS | 5 +++++ Objects/typeobject.c | 39 ++++++++++++++++++++++++++++++++------- 5 files changed, 78 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index a02c5f7..ccfd1e9 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4,6 +4,8 @@ For now, tests just new or changed functionality. """ +import copy +import pickle import sys import unittest from test import test_support @@ -35,6 +37,17 @@ class BufferTests(unittest.TestCase): buf = buffer(data, sys.maxsize, sys.maxsize) self.assertEqual(buf[:4096], "") + def test_copy(self): + buf = buffer(b'abc') + with self.assertRaises(TypeError): + copy.copy(buf) + + def test_pickle(self): + buf = buffer(b'abc') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.assertRaises(TypeError): + pickle.dumps(buf, proto) + def test_main(): with test_support.check_py3k_warnings(("buffer.. not supported", diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index e2eec70..181af99 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -2,6 +2,7 @@ # Copyright (C) 2001,2002 Python Software Foundation # csv package unit tests +import copy import sys import os import unittest @@ -10,6 +11,7 @@ import tempfile import csv import gc import io +import pickle from test import test_support class Test_Csv(unittest.TestCase): @@ -466,6 +468,17 @@ class TestDialectRegistry(unittest.TestCase): self.assertRaises(TypeError, csv.reader, [], quoting = -1) self.assertRaises(TypeError, csv.reader, [], quoting = 100) + def test_copy(self): + for name in csv.list_dialects(): + dialect = csv.get_dialect(name) + self.assertRaises(TypeError, copy.copy, dialect) + + def test_pickle(self): + for name in csv.list_dialects(): + dialect = csv.get_dialect(name) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.assertRaises(TypeError, pickle.dumps, dialect, proto) + class TestCsvBase(unittest.TestCase): def readerAssertEqual(self, input, expected_result): fd, name = tempfile.mkstemp() diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index f14bafd..bc83247 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -10,6 +10,8 @@ import weakref import array from test import test_support import io +import copy +import pickle class AbstractMemoryTests: @@ -354,6 +356,19 @@ class BytesMemorySliceSliceTest(unittest.TestCase, #pass +class OtherTest(unittest.TestCase): + def test_copy(self): + m = memoryview(b'abc') + with self.assertRaises(TypeError): + copy.copy(m) + + def test_pickle(self): + m = memoryview(b'abc') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.assertRaises(TypeError): + pickle.dumps(m, proto) + + def test_main(): test_support.run_unittest(__name__) diff --git a/Misc/NEWS b/Misc/NEWS index 48f5ddf..4f3dd83 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,11 @@ What's New in Python 2.7.12? Core and Builtins ----------------- +- Issue #22995: Instances of extension types with a state that aren't + subclasses of list or dict and haven't implemented any pickle-related + methods (__reduce__, __reduce_ex__, __getnewargs__, __getnewargs_ex__, + or __getstate__), can no longer be pickled. Including memoryview. + - Issue #20440: Massive replacing unsafe attribute setting code with special macro Py_SETREF. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 91709bc..a6f3caa 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3207,6 +3207,7 @@ reduce_2(PyObject *obj) PyObject *slots = NULL, *listitems = NULL, *dictitems = NULL; PyObject *copyreg = NULL, *newobj = NULL, *res = NULL; Py_ssize_t i, n; + int required_state = 0; cls = PyObject_GetAttrString(obj, "__class__"); if (cls == NULL) @@ -3214,7 +3215,7 @@ reduce_2(PyObject *obj) if (PyType_Check(cls) && ((PyTypeObject *)cls)->tp_new == NULL) { PyErr_Format(PyExc_TypeError, - "can't pickle %s objects", + "can't pickle %.200s objects", ((PyTypeObject *)cls)->tp_name); return NULL; } @@ -3223,7 +3224,9 @@ reduce_2(PyObject *obj) if (getnewargs != NULL) { args = PyObject_CallObject(getnewargs, NULL); Py_DECREF(getnewargs); - if (args != NULL && !PyTuple_Check(args)) { + if (args == NULL) + goto end; + if (!PyTuple_Check(args)) { PyErr_Format(PyExc_TypeError, "__getnewargs__ should return a tuple, " "not '%.200s'", Py_TYPE(args)->tp_name); @@ -3232,10 +3235,8 @@ reduce_2(PyObject *obj) } else { PyErr_Clear(); - args = PyTuple_New(0); + required_state = !PyList_Check(obj) && !PyDict_Check(obj); } - if (args == NULL) - goto end; getstate = PyObject_GetAttrString(obj, "__getstate__"); if (getstate != NULL) { @@ -3246,6 +3247,14 @@ reduce_2(PyObject *obj) } else { PyErr_Clear(); + + if (required_state && obj->ob_type->tp_itemsize) { + PyErr_Format(PyExc_TypeError, + "can't pickle %.200s objects", + Py_TYPE(obj)->tp_name); + goto end; + } + state = PyObject_GetAttrString(obj, "__dict__"); if (state == NULL) { PyErr_Clear(); @@ -3255,8 +3264,24 @@ reduce_2(PyObject *obj) names = slotnames(cls); if (names == NULL) goto end; + assert(names == Py_None || PyList_Check(names)); + if (required_state) { + Py_ssize_t basicsize = PyBaseObject_Type.tp_basicsize; + if (obj->ob_type->tp_dictoffset) + basicsize += sizeof(PyObject *); + if (obj->ob_type->tp_weaklistoffset) + basicsize += sizeof(PyObject *); + if (names != Py_None) + basicsize += sizeof(PyObject *) * Py_SIZE(names); + if (obj->ob_type->tp_basicsize > basicsize) { + PyErr_Format(PyExc_TypeError, + "can't pickle %.200s objects", + Py_TYPE(obj)->tp_name); + goto end; + } + } + if (names != Py_None) { - assert(PyList_Check(names)); slots = PyDict_New(); if (slots == NULL) goto end; @@ -3318,7 +3343,7 @@ reduce_2(PyObject *obj) if (newobj == NULL) goto end; - n = PyTuple_GET_SIZE(args); + n = args ? PyTuple_GET_SIZE(args) : 0; args2 = PyTuple_New(n+1); if (args2 == NULL) goto end; -- cgit v0.12