diff options
author | Jeremy Hylton <jeremy@alum.mit.edu> | 2001-10-12 04:11:06 (GMT) |
---|---|---|
committer | Jeremy Hylton <jeremy@alum.mit.edu> | 2001-10-12 04:11:06 (GMT) |
commit | a0fb177be8d04a6c393cdf4078ce58bfa76bc26d (patch) | |
tree | 2785f7018fb64036dd9640575ebb2460284f5267 /Modules/cPickle.c | |
parent | fc57ccb98248b4a8f4ba4debdf3791970b136c4d (diff) | |
download | cpython-a0fb177be8d04a6c393cdf4078ce58bfa76bc26d.zip cpython-a0fb177be8d04a6c393cdf4078ce58bfa76bc26d.tar.gz cpython-a0fb177be8d04a6c393cdf4078ce58bfa76bc26d.tar.bz2 |
Progress on SF bug #466175 and general cleanup.
Add a fast_container member to Picklerobject. If fast is true, then
fast_container counts the depth of nested container calls. If the
depth exceeds FAST_LIMIT (2000), the fast flag is ignored and the
normal checks occur. This approach is much like the approach for
prevent stack overflow for comparison and reprs of recursive objects
(e.g. [[...]]).
- Fast container used for save_list(), save_dict(), and
save_inst().
XXX Not clear which other save_xxx() functions should use it.
Make Picklerobject into new-style types, using PyObject_GenericGetAttr()
and PyObject_GenericSetAttr().
- Use PyMemberDef for binary and fast members
- Use PyGetSetDef for persistent_id, inst_persistent_id, memo, and
PicklingError.
XXX Not all of these seem like they need to use getset, but it's
not clear why the old getattr() and setattr() had such odd
semantics. One change is that the getvalue() attribute will
exist on all Picklers, not just list-based picklers; I think
this is a more rationale interface.
There is a long laundry list of other changes:
- Remove unused #defines for PyList_SET_ITEM() etc.
- Make some of the indentation consistent
- Replace uses of cPickle_PyMapping_HasKey() where the first
argument is self->memo with calls to PyDict_GetItem(), because
self->memo must be a dictionary.
- Don't bother to check if cPickle_PyMapping_HasKey() returns < 0,
because it can only return 0 or 1.
- Replace uses of PyObject_CallObject() with PyObject_Call(), when
we can guarantee that the argument tuple is really a tuple.
Performance impacts of these changes:
- 5% speedup for normal pickling
- No change to fast-mode pickling.
XXX Really need tests for all the features in cPickle that aren't in
pickle.
Diffstat (limited to 'Modules/cPickle.c')
-rw-r--r-- | Modules/cPickle.c | 384 |
1 files changed, 197 insertions, 187 deletions
diff --git a/Modules/cPickle.c b/Modules/cPickle.c index 60aee6b..eaead3f 100644 --- a/Modules/cPickle.c +++ b/Modules/cPickle.c @@ -54,6 +54,7 @@ static char cPickle_module_documentation[] = #include "Python.h" #include "cStringIO.h" +#include "structmember.h" #ifndef Py_eval_input #include <graminit.h> @@ -144,22 +145,6 @@ static PyObject *__class___str, *__getinitargs___str, *__dict___str, *read_str, *readline_str, *__main___str, *__basicnew___str, *copy_reg_str, *dispatch_table_str, *safe_constructors_str; -#ifndef PyList_SET_ITEM -#define PyList_SET_ITEM(op, i, v) (((PyListObject *)(op))->ob_item[i] = (v)) -#endif -#ifndef PyList_GET_SIZE -#define PyList_GET_SIZE(op) (((PyListObject *)(op))->ob_size) -#endif -#ifndef PyTuple_SET_ITEM -#define PyTuple_SET_ITEM(op, i, v) (((PyTupleObject *)(op))->ob_item[i] = (v)) -#endif -#ifndef PyTuple_GET_SIZE -#define PyTuple_GET_SIZE(op) (((PyTupleObject *)(op))->ob_size) -#endif -#ifndef PyString_GET_SIZE -#define PyString_GET_SIZE(op) (((PyStringObject *)(op))->ob_size) -#endif - /************************************************************************* Internal Data type for pickle data. */ @@ -318,22 +303,25 @@ Pdata_popList(Pdata *self, int start) { } typedef struct Picklerobject { - PyObject_HEAD - FILE *fp; - PyObject *write; - PyObject *file; - PyObject *memo; - PyObject *arg; - PyObject *pers_func; - PyObject *inst_pers_func; - int bin; - int fast; /* Fast mode doesn't save in memo, don't use if circ ref */ - int (*write_func)(struct Picklerobject *, char *, int); - char *write_buf; - int buf_size; - PyObject *dispatch_table; + PyObject_HEAD + FILE *fp; + PyObject *write; + PyObject *file; + PyObject *memo; + PyObject *arg; + PyObject *pers_func; + PyObject *inst_pers_func; + int bin; + int fast; /* Fast mode doesn't save in memo, don't use if circ ref */ + int (*write_func)(struct Picklerobject *, char *, int); + char *write_buf; + int buf_size; + PyObject *dispatch_table; + int fast_container; /* count nested container dumps */ } Picklerobject; +#define FAST_LIMIT 2000 + staticforward PyTypeObject Picklertype; typedef struct Unpicklerobject { @@ -482,7 +470,7 @@ write_other(Picklerobject *self, char *s, int n) { /* object with write method */ ARG_TUP(self, py_str); if (self->arg) { - junk = PyObject_CallObject(self->write, self->arg); + junk = PyObject_Call(self->write, self->arg, NULL); FREE_ARG_TUP(self); } if (junk) Py_DECREF(junk); @@ -612,7 +600,7 @@ read_other(Unpicklerobject *self, char **s, int n) { ARG_TUP(self, bytes); if (self->arg) { - str = PyObject_CallObject(self->read, self->arg); + str = PyObject_Call(self->read, self->arg, NULL); FREE_ARG_TUP(self); } if (! str) return -1; @@ -713,7 +701,7 @@ get(Picklerobject *self, PyObject *id) { static int put(Picklerobject *self, PyObject *ob) { - if (ob->ob_refcnt < 2 || self->fast) + if (ob->ob_refcnt < 2 || self->fast) return 0; return put2(self, ob); @@ -728,7 +716,8 @@ put2(Picklerobject *self, PyObject *ob) { int res = -1; PyObject *py_ob_id = 0, *memo_len = 0, *t = 0; - if (self->fast) return 0; + if (self->fast) + return 0; if ((p = PyDict_Size(self->memo)) < 0) goto finally; @@ -1296,7 +1285,7 @@ err: static int save_tuple(Picklerobject *self, PyObject *args) { PyObject *element = 0, *py_tuple_id = 0; - int len, i, has_key, res = -1; + int len, i, res = -1; static char tuple = TUPLE; @@ -1318,10 +1307,7 @@ save_tuple(Picklerobject *self, PyObject *args) { goto finally; if (len) { - if ((has_key = cPickle_PyMapping_HasKey(self->memo, py_tuple_id)) < 0) - goto finally; - - if (has_key) { + if (PyDict_GetItem(self->memo, py_tuple_id)) { if (self->bin) { static char pop_mark = POP_MARK; @@ -1371,11 +1357,16 @@ save_empty_tuple(Picklerobject *self, PyObject *args) { static int save_list(Picklerobject *self, PyObject *args) { PyObject *element = 0; - int s_len, len, i, using_appends, res = -1; + int s_len, len, i, using_appends, res = -1, unfast = 0; char s[3]; static char append = APPEND, appends = APPENDS; + if (self->fast && self->fast_container++ > FAST_LIMIT) { + self->fast = 0; + unfast = 1; + } + if (self->bin) { s[0] = EMPTY_LIST; s_len = 1; @@ -1426,6 +1417,11 @@ save_list(Picklerobject *self, PyObject *args) { res = 0; finally: + if (self->fast || unfast) { + self->fast_container--; + if (unfast && self->fast_container < FAST_LIMIT) + self->fast = 1; + } return res; } @@ -1434,11 +1430,16 @@ finally: static int save_dict(Picklerobject *self, PyObject *args) { PyObject *key = 0, *value = 0; - int i, len, res = -1, using_setitems; + int i, len, res = -1, using_setitems, unfast = 0; char s[3]; static char setitem = SETITEM, setitems = SETITEMS; + if (self->fast && self->fast_container++ > FAST_LIMIT) { + self->fast = 0; + unfast = 1; + } + if (self->bin) { s[0] = EMPTY_DICT; len = 1; @@ -1490,6 +1491,12 @@ save_dict(Picklerobject *self, PyObject *args) { res = 0; finally: + if (self->fast || unfast) { + self->fast_container--; + if (unfast && self->fast_container < FAST_LIMIT) + self->fast = 1; + } + return res; } @@ -1500,10 +1507,15 @@ save_inst(Picklerobject *self, PyObject *args) { PyObject *class = 0, *module = 0, *name = 0, *state = 0, *getinitargs_func = 0, *getstate_func = 0, *class_args = 0; char *module_str, *name_str; - int module_size, name_size, res = -1; + int module_size, name_size, res = -1, unfast = 0; static char inst = INST, obj = OBJ, build = BUILD; + if (self->fast && self->fast_container++ > FAST_LIMIT) { + self->fast = 0; + unfast = 1; + } + if ((*self->write_func)(self, &MARKv, 1) < 0) goto finally; @@ -1520,7 +1532,7 @@ save_inst(Picklerobject *self, PyObject *args) { int i, len; UNLESS (class_args = - PyObject_CallObject(getinitargs_func, empty_tuple)) + PyObject_Call(getinitargs_func, empty_tuple, NULL)) goto finally; if ((len = PyObject_Size(class_args)) < 0) @@ -1579,7 +1591,7 @@ save_inst(Picklerobject *self, PyObject *args) { } if ((getstate_func = PyObject_GetAttr(args, __getstate___str))) { - UNLESS (state = PyObject_CallObject(getstate_func, empty_tuple)) + UNLESS (state = PyObject_Call(getstate_func, empty_tuple, NULL)) goto finally; } else { @@ -1610,6 +1622,12 @@ save_inst(Picklerobject *self, PyObject *args) { res = 0; finally: + if (self->fast || unfast) { + self->fast_container--; + if (unfast && self->fast_container < FAST_LIMIT) + self->fast = 1; + } + Py_XDECREF(module); Py_XDECREF(class); Py_XDECREF(state); @@ -1709,7 +1727,7 @@ save_pers(Picklerobject *self, PyObject *args, PyObject *f) { Py_INCREF(args); ARG_TUP(self, args); if (self->arg) { - pid = PyObject_CallObject(f, self->arg); + pid = PyObject_Call(f, self->arg, NULL); FREE_ARG_TUP(self); } if (! pid) return -1; @@ -1860,15 +1878,10 @@ save(Picklerobject *self, PyObject *args, int pers_save) { } if (args->ob_refcnt > 1) { - int has_key; - UNLESS (py_ob_id = PyLong_FromVoidPtr(args)) goto finally; - if ((has_key = cPickle_PyMapping_HasKey(self->memo, py_ob_id)) < 0) - goto finally; - - if (has_key) { + if (PyDict_GetItem(self->memo, py_ob_id)) { if (get(self, py_ob_id) < 0) goto finally; @@ -1960,7 +1973,7 @@ save(Picklerobject *self, PyObject *args, int pers_save) { Py_INCREF(args); ARG_TUP(self, args); if (self->arg) { - t = PyObject_CallObject(__reduce__, self->arg); + t = PyObject_Call(__reduce__, self->arg, NULL); FREE_ARG_TUP(self); } if (! t) goto finally; @@ -1969,7 +1982,7 @@ save(Picklerobject *self, PyObject *args, int pers_save) { PyErr_Clear(); if ((__reduce__ = PyObject_GetAttr(args, __reduce___str))) { - UNLESS (t = PyObject_CallObject(__reduce__, empty_tuple)) + UNLESS (t = PyObject_Call(__reduce__, empty_tuple, NULL)) goto finally; } else { @@ -2044,10 +2057,12 @@ dump(Picklerobject *self, PyObject *args) { static PyObject * Pickle_clear_memo(Picklerobject *self, PyObject *args) { - if (args && ! PyArg_ParseTuple(args,":clear_memo")) return NULL; - if (self->memo) PyDict_Clear(self->memo); - Py_INCREF(Py_None); - return Py_None; + if (!PyArg_ParseTuple(args,":clear_memo")) + return NULL; + if (self->memo) + PyDict_Clear(self->memo); + Py_INCREF(Py_None); + return Py_None; } static PyObject * @@ -2058,14 +2073,16 @@ Pickle_getvalue(Picklerobject *self, PyObject *args) { char *s, *p, *have_get; Pdata *data; - if (args && ! PyArg_ParseTuple(args,"|i:getvalue",&clear)) return NULL; + /* Can be called by Python code or C code */ + if (args && !PyArg_ParseTuple(args, "|i:getvalue", &clear)) + return NULL; /* Check to make sure we are based on a list */ if (! Pdata_Check(self->file)) { PyErr_SetString(PicklingError, - "Attempt to getvalue a non-list-based pickler"); + "Attempt to getvalue() a non-list-based pickler"); return NULL; - } + } /* flush write buffer */ if (write_other(self, NULL, 0) < 0) return NULL; @@ -2198,6 +2215,7 @@ Pickler_dump(Picklerobject *self, PyObject *args) { if (get) return Pickle_getvalue(self, NULL); + /* XXX Why does dump() return self? */ Py_INCREF(self); return (PyObject*)self; } @@ -2232,6 +2250,7 @@ newPicklerobject(PyObject *file, int bin) { self->write_buf = NULL; self->bin = bin; self->fast = 0; + self->fast_container = 0; self->buf_size = 0; self->dispatch_table = NULL; @@ -2303,16 +2322,15 @@ err: static PyObject * get_Pickler(PyObject *self, PyObject *args) { - PyObject *file=NULL; - int bin; + PyObject *file = NULL; + int bin = 1; - bin=1; - if (! PyArg_ParseTuple(args, "|i:Pickler", &bin)) { + if (!PyArg_ParseTuple(args, "|i:Pickler", &bin)) { PyErr_Clear(); - bin=0; - if (! PyArg_ParseTuple(args, "O|i:Pickler", &file, &bin)) - return NULL; - } + bin = 0; + if (!PyArg_ParseTuple(args, "O|i:Pickler", &file, &bin)) + return NULL; + } return (PyObject *)newPicklerobject(file, bin); } @@ -2334,134 +2352,128 @@ Pickler_dealloc(Picklerobject *self) { PyObject_Del(self); } - static PyObject * -Pickler_getattr(Picklerobject *self, char *name) { - - switch (*name) { - case 'p': - if (strcmp(name, "persistent_id") == 0) { - if (!self->pers_func) { - PyErr_SetString(PyExc_AttributeError, name); - return NULL; - } - - Py_INCREF(self->pers_func); - return self->pers_func; - } - break; - case 'm': - if (strcmp(name, "memo") == 0) { - if (!self->memo) { - PyErr_SetString(PyExc_AttributeError, name); - return NULL; - } - - Py_INCREF(self->memo); - return self->memo; - } - break; - case 'P': - if (strcmp(name, "PicklingError") == 0) { - Py_INCREF(PicklingError); - return PicklingError; - } - break; - case 'b': - if (strcmp(name, "binary")==0) - return PyInt_FromLong(self->bin); - break; - case 'f': - if (strcmp(name, "fast")==0) - return PyInt_FromLong(self->fast); - break; - case 'g': - if (strcmp(name, "getvalue")==0 && ! Pdata_Check(self->file)) { - PyErr_SetString(PyExc_AttributeError, name); - return NULL; - } - break; - } - return Py_FindMethod(Pickler_methods, (PyObject *)self, name); +Pickler_get_pers_func(Picklerobject *p) +{ + if (p->pers_func == NULL) + PyErr_SetString(PyExc_AttributeError, "persistent_id"); + else + Py_INCREF(p->pers_func); + return p->pers_func; } - -int -Pickler_setattr(Picklerobject *self, char *name, PyObject *value) { - - if (! value) { +static int +Pickler_set_pers_func(Picklerobject *p, PyObject *v) +{ + if (v == NULL) { PyErr_SetString(PyExc_TypeError, - "attribute deletion is not supported"); + "attribute deletion is not supported"); return -1; } + Py_XDECREF(p->pers_func); + Py_INCREF(v); + p->pers_func = v; + return 0; +} - if (strcmp(name, "persistent_id") == 0) { - Py_XDECREF(self->pers_func); - self->pers_func = value; - Py_INCREF(value); - return 0; - } - - if (strcmp(name, "inst_persistent_id") == 0) { - Py_XDECREF(self->inst_pers_func); - self->inst_pers_func = value; - Py_INCREF(value); - return 0; +static int +Pickler_set_inst_pers_func(Picklerobject *p, PyObject *v) +{ + if (v == NULL) { + PyErr_SetString(PyExc_TypeError, + "attribute deletion is not supported"); + return -1; } + Py_XDECREF(p->inst_pers_func); + Py_INCREF(v); + p->inst_pers_func = v; + return 0; +} - if (strcmp(name, "memo") == 0) { - if (! PyDict_Check(value)) { - PyErr_SetString(PyExc_TypeError, "memo must be a dictionary"); - return -1; - } - Py_XDECREF(self->memo); - self->memo = value; - Py_INCREF(value); - return 0; - } +static PyObject * +Pickler_get_memo(Picklerobject *p) +{ + if (p->memo == NULL) + PyErr_SetString(PyExc_AttributeError, "memo"); + else + Py_INCREF(p->memo); + return p->memo; +} - if (strcmp(name, "binary")==0) { - self->bin=PyObject_IsTrue(value); - return 0; +static int +Pickler_set_memo(Picklerobject *p, PyObject *v) +{ + if (v == NULL) { + PyErr_SetString(PyExc_TypeError, + "attribute deletion is not supported"); + return -1; } - - if (strcmp(name, "fast")==0) { - self->fast=PyObject_IsTrue(value); - return 0; + if (!PyDict_Check(v)) { + PyErr_SetString(PyExc_TypeError, "memo must be a dictionary"); + return -1; } + Py_XDECREF(p->memo); + Py_INCREF(v); + p->memo = v; + return 0; +} - PyErr_SetString(PyExc_AttributeError, name); - return -1; +static PyObject * +Pickler_get_error(Picklerobject *p) +{ + /* why is this an attribute on the Pickler? */ + Py_INCREF(PicklingError); + return PicklingError; } +static PyMemberDef Pickler_members[] = { + {"binary", T_INT, offsetof(Picklerobject, bin)}, + {"fast", T_INT, offsetof(Picklerobject, fast)}, +}; + +static PyGetSetDef Pickler_getsets[] = { + {"persistent_id", (getter)Pickler_get_pers_func, + (setter)Pickler_set_pers_func}, + {"inst_persistent_id", NULL, (setter)Pickler_set_inst_pers_func}, + {"memo", (getter)Pickler_get_memo, (setter)Pickler_set_memo}, + {"PicklingError", (getter)Pickler_get_error, NULL} +}; static char Picklertype__doc__[] = -"Objects that know how to pickle objects\n" -; +"Objects that know how to pickle objects\n"; static PyTypeObject Picklertype = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "Pickler", /*tp_name*/ - sizeof(Picklerobject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - /* methods */ - (destructor)Pickler_dealloc, /*tp_dealloc*/ - (printfunc)0, /*tp_print*/ - (getattrfunc)Pickler_getattr, /*tp_getattr*/ - (setattrfunc)Pickler_setattr, /*tp_setattr*/ - (cmpfunc)0, /*tp_compare*/ - (reprfunc)0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - (hashfunc)0, /*tp_hash*/ - (ternaryfunc)0, /*tp_call*/ - (reprfunc)0, /*tp_str*/ - - /* Space for future expansion */ - 0L,0L,0L,0L, - Picklertype__doc__ /* Documentation string */ + sizeof(Picklerobject), /*tp_basicsize*/ + 0, + (destructor)Pickler_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + PyObject_GenericSetAttr, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + Picklertype__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + Pickler_methods, /* tp_methods */ + Pickler_members, /* tp_members */ + Pickler_getsets, /* tp_getset */ }; static PyObject * @@ -2994,10 +3006,8 @@ Instance_New(PyObject *cls, PyObject *args) { else goto err; } - - if ((has_key = cPickle_PyMapping_HasKey(safe_constructors, cls)) < 0) - goto err; - + /* Is safe_constructors always a dict? */ + has_key = cPickle_PyMapping_HasKey(safe_constructors, cls); if (!has_key) if (!(safe = PyObject_GetAttr(cls, __safe_for_unpickling___str)) || !PyObject_IsTrue(safe)) { @@ -3136,7 +3146,7 @@ load_persid(Unpicklerobject *self) { else { ARG_TUP(self, pid); if (self->arg) { - pid = PyObject_CallObject(self->pers_func, self->arg); + pid = PyObject_Call(self->pers_func, self->arg, NULL); FREE_ARG_TUP(self); } } @@ -3171,7 +3181,7 @@ load_binpersid(Unpicklerobject *self) { else { ARG_TUP(self, pid); if (self->arg) { - pid = PyObject_CallObject(self->pers_func, self->arg); + pid = PyObject_Call(self->pers_func, self->arg, NULL); FREE_ARG_TUP(self); } if (! pid) return -1; @@ -3424,7 +3434,7 @@ do_append(Unpicklerobject *self, int x) { junk=0; ARG_TUP(self, value); if (self->arg) { - junk = PyObject_CallObject(append_method, self->arg); + junk = PyObject_Call(append_method, self->arg, NULL); FREE_ARG_TUP(self); } if (! junk) { @@ -3505,7 +3515,7 @@ load_build(Unpicklerobject *self) { if ((__setstate__ = PyObject_GetAttr(inst, __setstate___str))) { ARG_TUP(self, value); if (self->arg) { - junk = PyObject_CallObject(__setstate__, self->arg); + junk = PyObject_Call(__setstate__, self->arg, NULL); FREE_ARG_TUP(self); } Py_DECREF(__setstate__); @@ -4322,9 +4332,9 @@ Unpickler_setattr(Unpicklerobject *self, char *name, PyObject *value) { } if (strcmp(name, "memo") == 0) { - if (! PyDict_Check(value)) { - PyErr_SetString(PyExc_TypeError, "memo must be a dictionary"); - return -1; + if (!PyDict_Check(value)) { + PyErr_SetString(PyExc_TypeError, "memo must be a dictionary"); + return -1; } Py_XDECREF(self->memo); self->memo = value; @@ -4499,7 +4509,7 @@ static int init_stuff(PyObject *module_dict) { PyObject *copy_reg, *t, *r; -#define INIT_STR(S) UNLESS(S ## _str=PyString_FromString(#S)) return -1; +#define INIT_STR(S) UNLESS(S ## _str=PyString_InternFromString(#S)) return -1; INIT_STR(__class__); INIT_STR(__getinitargs__); @@ -4529,7 +4539,7 @@ init_stuff(PyObject *module_dict) { return -1; UNLESS (safe_constructors = PyObject_GetAttr(copy_reg, - safe_constructors_str)) + safe_constructors_str)) return -1; Py_DECREF(copy_reg); |