diff options
| -rw-r--r-- | Lib/test/pickletester.py | 31 | ||||
| -rw-r--r-- | Lib/test/test_cpickle.py | 17 | ||||
| -rw-r--r-- | Lib/test/test_pickle.py | 20 | ||||
| -rw-r--r-- | Misc/NEWS | 2 | ||||
| -rw-r--r-- | Modules/cPickle.c | 270 | ||||
| -rw-r--r-- | Modules/cStringIO.c | 7 |
6 files changed, 217 insertions, 130 deletions
diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index c321f8e..f735e95 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -6,7 +6,8 @@ import cStringIO import pickletools import copy_reg -from test.test_support import TestFailed, have_unicode, TESTFN +from test.test_support import (TestFailed, have_unicode, TESTFN, _2G, _1M, + precisionbigmemtest) # Tests that try a number of pickle protocols should have a # for proto in protocols: @@ -1280,3 +1281,31 @@ class AbstractPicklerUnpicklerObjectTests(unittest.TestCase): f.write(pickled2) f.seek(0) self.assertEqual(unpickler.load(), data2) + +class BigmemPickleTests(unittest.TestCase): + + # Memory requirements: 1 byte per character for input strings, 1 byte + # for pickled data, 1 byte for unpickled strings, 1 byte for internal + # buffer and 1 byte of free space for resizing of internal buffer. + + @precisionbigmemtest(size=_2G + 100*_1M, memuse=5) + def test_huge_strlist(self, size): + chunksize = 2**20 + data = [] + while size > chunksize: + data.append('x' * chunksize) + size -= chunksize + chunksize += 1 + data.append('y' * size) + + try: + for proto in protocols: + try: + pickled = self.dumps(data, proto) + res = self.loads(pickled) + self.assertEqual(res, data) + finally: + res = None + pickled = None + finally: + data = None diff --git a/Lib/test/test_cpickle.py b/Lib/test/test_cpickle.py index c240669..beea318 100644 --- a/Lib/test/test_cpickle.py +++ b/Lib/test/test_cpickle.py @@ -1,7 +1,9 @@ import cPickle, unittest from cStringIO import StringIO -from test.pickletester import AbstractPickleTests, AbstractPickleModuleTests -from test.pickletester import AbstractPicklerUnpicklerObjectTests +from test.pickletester import (AbstractPickleTests, + AbstractPickleModuleTests, + AbstractPicklerUnpicklerObjectTests, + BigmemPickleTests) from test import test_support class cPickleTests(AbstractPickleTests, AbstractPickleModuleTests): @@ -101,6 +103,16 @@ class cPicklePicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests): pickler_class = cPickle.Pickler unpickler_class = cPickle.Unpickler +class cPickleBigmemPickleTests(BigmemPickleTests): + + def dumps(self, arg, proto=0, fast=0): + # Ignore fast + return cPickle.dumps(arg, proto) + + def loads(self, buf): + # Ignore fast + return cPickle.loads(buf) + class Node(object): pass @@ -133,6 +145,7 @@ def test_main(): cPickleFastPicklerTests, cPickleDeepRecursive, cPicklePicklerUnpicklerObjectTests, + cPickleBigmemPickleTests, ) if __name__ == "__main__": diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 6c83811..c312649 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -3,10 +3,11 @@ from cStringIO import StringIO from test import test_support -from test.pickletester import AbstractPickleTests -from test.pickletester import AbstractPickleModuleTests -from test.pickletester import AbstractPersistentPicklerTests -from test.pickletester import AbstractPicklerUnpicklerObjectTests +from test.pickletester import (AbstractPickleTests, + AbstractPickleModuleTests, + AbstractPersistentPicklerTests, + AbstractPicklerUnpicklerObjectTests, + BigmemPickleTests) class PickleTests(AbstractPickleTests, AbstractPickleModuleTests): @@ -66,6 +67,16 @@ class PicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests): pickler_class = pickle.Pickler unpickler_class = pickle.Unpickler +class PickleBigmemPickleTests(BigmemPickleTests): + + def dumps(self, arg, proto=0, fast=0): + # Ignore fast + return pickle.dumps(arg, proto) + + def loads(self, buf): + # Ignore fast + return pickle.loads(buf) + def test_main(): test_support.run_unittest( @@ -73,6 +84,7 @@ def test_main(): PicklerTests, PersPicklerTests, PicklerUnpicklerObjectTests, + PickleBigmemPickleTests, ) test_support.run_doctest(pickle) @@ -202,6 +202,8 @@ Core and Builtins Library ------- +- Issue #13555: cPickle now supports files larger than 2 GiB. + - Issue #17052: unittest discovery should use self.testLoader. - Issue #4591: Uid and gid values larger than 2**31 are supported now. diff --git a/Modules/cPickle.c b/Modules/cPickle.c index 48b5075..e73bac1 100644 --- a/Modules/cPickle.c +++ b/Modules/cPickle.c @@ -139,15 +139,15 @@ static PyObject *__class___str, *__getinitargs___str, *__dict___str, typedef struct { PyObject_HEAD - int length; /* number of initial slots in data currently used */ - int size; /* number of slots in data allocated */ + Py_ssize_t length; /* number of initial slots in data currently used */ + Py_ssize_t size; /* number of slots in data allocated */ PyObject **data; } Pdata; static void Pdata_dealloc(Pdata *self) { - int i; + Py_ssize_t i; PyObject **p; for (i = self->length, p = self->data; --i >= 0; p++) { @@ -193,9 +193,9 @@ stackUnderflow(void) * number of items, this is a (non-erroneous) NOP. */ static int -Pdata_clear(Pdata *self, int clearto) +Pdata_clear(Pdata *self, Py_ssize_t clearto) { - int i; + Py_ssize_t i; PyObject **p; if (clearto < 0) return stackUnderflow(); @@ -214,18 +214,17 @@ Pdata_clear(Pdata *self, int clearto) static int Pdata_grow(Pdata *self) { - int bigger; - size_t nbytes; + Py_ssize_t bigger; + Py_ssize_t nbytes; + PyObject **tmp; - bigger = self->size << 1; - if (bigger <= 0) /* was 0, or new value overflows */ + if (self->size > (PY_SSIZE_T_MAX >> 1)) goto nomemory; - if ((int)(size_t)bigger != bigger) - goto nomemory; - nbytes = (size_t)bigger * sizeof(PyObject *); - if (nbytes / sizeof(PyObject *) != (size_t)bigger) + bigger = self->size << 1; + if (bigger > (PY_SSIZE_T_MAX / sizeof(PyObject *))) goto nomemory; + nbytes = bigger * sizeof(PyObject *); tmp = realloc(self->data, nbytes); if (tmp == NULL) goto nomemory; @@ -280,10 +279,10 @@ Pdata_grow(Pdata *self) static PyObject * -Pdata_popTuple(Pdata *self, int start) +Pdata_popTuple(Pdata *self, Py_ssize_t start) { PyObject *r; - int i, j, l; + Py_ssize_t i, j, l; l = self->length-start; r = PyTuple_New(l); @@ -297,10 +296,10 @@ Pdata_popTuple(Pdata *self, int start) } static PyObject * -Pdata_popList(Pdata *self, int start) +Pdata_popList(Pdata *self, Py_ssize_t start) { PyObject *r; - int i, j, l; + Py_ssize_t i, j, l; l=self->length-start; if (!( r=PyList_New(l))) return NULL; @@ -347,9 +346,9 @@ typedef struct Picklerobject { int bin; int fast; /* Fast mode doesn't save in memo, don't use if circ ref */ - int (*write_func)(struct Picklerobject *, const char *, Py_ssize_t); + Py_ssize_t (*write_func)(struct Picklerobject *, const char *, Py_ssize_t); char *write_buf; - int buf_size; + Py_ssize_t buf_size; PyObject *dispatch_table; int fast_container; /* count nested container dumps */ PyObject *fast_memo; @@ -373,12 +372,12 @@ typedef struct Unpicklerobject { PyObject *mark; PyObject *pers_func; PyObject *last_string; - int *marks; - int num_marks; - int marks_size; + Py_ssize_t *marks; + Py_ssize_t num_marks; + Py_ssize_t marks_size; Py_ssize_t (*read_func)(struct Unpicklerobject *, char **, Py_ssize_t); Py_ssize_t (*readline_func)(struct Unpicklerobject *, char **); - int buf_size; + Py_ssize_t buf_size; char *buf; PyObject *find_class; } Unpicklerobject; @@ -424,7 +423,7 @@ cPickle_ErrFormat(PyObject *ErrType, char *stringformat, char *format, ...) return NULL; } -static int +static Py_ssize_t write_file(Picklerobject *self, const char *s, Py_ssize_t n) { size_t nbyteswritten; @@ -433,11 +432,6 @@ write_file(Picklerobject *self, const char *s, Py_ssize_t n) return 0; } - if (n > INT_MAX) { - /* String too large */ - return -1; - } - PyFile_IncUseCount((PyFileObject *)self->file); Py_BEGIN_ALLOW_THREADS nbyteswritten = fwrite(s, sizeof(char), n, self->fp); @@ -448,40 +442,44 @@ write_file(Picklerobject *self, const char *s, Py_ssize_t n) return -1; } - return (int)n; + return n; } -static int +static Py_ssize_t write_cStringIO(Picklerobject *self, const char *s, Py_ssize_t n) { + Py_ssize_t len = n; + if (s == NULL) { return 0; } + while (n > INT_MAX) { + if (PycStringIO->cwrite((PyObject *)self->file, s, INT_MAX) != INT_MAX) { + return -1; + } + n -= INT_MAX; + } + if (PycStringIO->cwrite((PyObject *)self->file, s, n) != n) { return -1; } - return (int)n; + return len; } -static int +static Py_ssize_t write_none(Picklerobject *self, const char *s, Py_ssize_t n) { if (s == NULL) return 0; - if (n > INT_MAX) return -1; - return (int)n; + return n; } -static int -write_other(Picklerobject *self, const char *s, Py_ssize_t _n) +static Py_ssize_t +write_other(Picklerobject *self, const char *s, Py_ssize_t n) { PyObject *py_str = 0, *junk = 0; - int n; - if (_n > INT_MAX) - return -1; - n = (int)_n; if (s == NULL) { if (!( self->buf_size )) return 0; py_str = PyString_FromStringAndSize(self->write_buf, @@ -490,7 +488,7 @@ write_other(Picklerobject *self, const char *s, Py_ssize_t _n) return -1; } else { - if (self->buf_size && (n + self->buf_size) > WRITE_BUF_SIZE) { + if (self->buf_size && n > WRITE_BUF_SIZE - self->buf_size) { if (write_other(self, NULL, 0) < 0) return -1; } @@ -531,7 +529,7 @@ read_file(Unpicklerobject *self, char **s, Py_ssize_t n) size_t nbytesread; if (self->buf_size == 0) { - int size; + Py_ssize_t size; size = ((n < 32) ? 32 : n); if (!( self->buf = (char *)malloc(size))) { @@ -575,7 +573,7 @@ read_file(Unpicklerobject *self, char **s, Py_ssize_t n) static Py_ssize_t readline_file(Unpicklerobject *self, char **s) { - int i; + Py_ssize_t i; if (self->buf_size == 0) { if (!( self->buf = (char *)malloc(40))) { @@ -587,7 +585,7 @@ readline_file(Unpicklerobject *self, char **s) i = 0; while (1) { - int bigger; + Py_ssize_t bigger; char *newbuf; for (; i < (self->buf_size - 1); i++) { if (feof(self->fp) || @@ -597,13 +595,13 @@ readline_file(Unpicklerobject *self, char **s) return i + 1; } } - bigger = self->buf_size << 1; - if (bigger <= 0) { /* overflow */ + if (self->buf_size < (PY_SSIZE_T_MAX >> 1)) { PyErr_NoMemory(); return -1; } + bigger = self->buf_size << 1; newbuf = (char *)realloc(self->buf, bigger); - if (!newbuf) { + if (newbuf == NULL) { PyErr_NoMemory(); return -1; } @@ -616,30 +614,63 @@ readline_file(Unpicklerobject *self, char **s) static Py_ssize_t read_cStringIO(Unpicklerobject *self, char **s, Py_ssize_t n) { - char *ptr; + Py_ssize_t len = n; + char *start, *end = NULL; - if (PycStringIO->cread((PyObject *)self->file, &ptr, n) != n) { - PyErr_SetNone(PyExc_EOFError); - return -1; + while (1) { + int k; + char *ptr; + if (n > INT_MAX) + k = INT_MAX; + else + k = (int)n; + if (PycStringIO->cread((PyObject *)self->file, &ptr, k) != k) { + PyErr_SetNone(PyExc_EOFError); + return -1; + } + if (end == NULL) + start = ptr; + else if (ptr != end) { + /* non-continuous area */ + return -1; + } + if (n <= INT_MAX) + break; + end = ptr + INT_MAX; + n -= INT_MAX; } - *s = ptr; + *s = start; - return n; + return len; } static Py_ssize_t readline_cStringIO(Unpicklerobject *self, char **s) { - Py_ssize_t n; - char *ptr; + Py_ssize_t n = 0; + char *start = NULL, *end = NULL; - if ((n = PycStringIO->creadline((PyObject *)self->file, &ptr)) < 0) { - return -1; + while (1) { + int k; + char *ptr; + if ((k = PycStringIO->creadline((PyObject *)self->file, &ptr)) < 0) { + return -1; + } + n += k; + if (end == NULL) + start = ptr; + else if (ptr != end) { + /* non-continuous area */ + return -1; + } + if (k == 0 || ptr[k - 1] == '\n') + break; + end = ptr + k; } - *s = ptr; + *s = start; return n; } @@ -700,7 +731,7 @@ readline_other(Unpicklerobject *self, char **s) * The caller is responsible for free()'ing the return value. */ static char * -pystrndup(const char *s, int n) +pystrndup(const char *s, Py_ssize_t n) { char *r = (char *)malloc(n+1); if (r == NULL) @@ -715,7 +746,7 @@ static int get(Picklerobject *self, PyObject *id) { PyObject *value, *mv; - long c_value; + Py_ssize_t c_value; char s[30]; size_t len; @@ -735,7 +766,8 @@ get(Picklerobject *self, PyObject *id) if (!self->bin) { s[0] = GET; - PyOS_snprintf(s + 1, sizeof(s) - 1, "%ld\n", c_value); + PyOS_snprintf(s + 1, sizeof(s) - 1, + "%" PY_FORMAT_SIZE_T "d\n", c_value); len = strlen(s); } else if (Pdata_Check(self->file)) { @@ -780,8 +812,7 @@ static int put2(Picklerobject *self, PyObject *ob) { char c_str[30]; - int p; - size_t len; + Py_ssize_t len, p; int res = -1; PyObject *py_ob_id = 0, *memo_len = 0, *t = 0; @@ -818,7 +849,8 @@ put2(Picklerobject *self, PyObject *ob) if (!self->bin) { c_str[0] = PUT; - PyOS_snprintf(c_str + 1, sizeof(c_str) - 1, "%d\n", p); + PyOS_snprintf(c_str + 1, sizeof(c_str) - 1, + "%" PY_FORMAT_SIZE_T "d\n", p); len = strlen(c_str); } else if (Pdata_Check(self->file)) { @@ -994,7 +1026,7 @@ save_int(Picklerobject *self, PyObject *args) { char c_str[32]; long l = PyInt_AS_LONG((PyIntObject *)args); - int len = 0; + Py_ssize_t len = 0; if (!self->bin #if SIZEOF_LONG > 4 @@ -1201,7 +1233,7 @@ done: static int save_string(Picklerobject *self, PyObject *args, int doput) { - int size, len; + Py_ssize_t size, len; PyObject *repr=0; if ((size = PyString_Size(args)) < 0) @@ -1448,7 +1480,7 @@ save_unicode(Picklerobject *self, PyObject *args, int doput) static int store_tuple_elements(Picklerobject *self, PyObject *t, int len) { - int i; + Py_ssize_t i; int res = -1; /* guilty until proved innocent */ assert(PyTuple_Size(t) == len); @@ -1477,7 +1509,7 @@ static int save_tuple(Picklerobject *self, PyObject *args) { PyObject *py_tuple_id = NULL; - int len, i; + Py_ssize_t len, i; int res = -1; static char tuple = TUPLE; @@ -1690,7 +1722,7 @@ save_list(Picklerobject *self, PyObject *args) { int res = -1; char s[3]; - int len; + Py_ssize_t len; PyObject *iter; if (self->fast && !fast_save_enter(self, args)) @@ -1943,7 +1975,7 @@ save_dict(Picklerobject *self, PyObject *args) { int res = -1; char s[3]; - int len; + Py_ssize_t len; if (self->fast && !fast_save_enter(self, args)) goto finally; @@ -2027,7 +2059,7 @@ save_inst(Picklerobject *self, PyObject *args) if ((getinitargs_func = PyObject_GetAttr(args, __getinitargs___str))) { PyObject *element = 0; - int i, len; + Py_ssize_t i, len; if (!( class_args = PyObject_Call(getinitargs_func, empty_tuple, NULL))) @@ -2289,7 +2321,8 @@ static int save_pers(Picklerobject *self, PyObject *args, PyObject *f) { PyObject *pid = 0; - int size, res = -1; + Py_ssize_t size; + int res = -1; static char persid = PERSID, binpersid = BINPERSID; @@ -2431,7 +2464,7 @@ save_reduce(Picklerobject *self, PyObject *args, PyObject *fn, PyObject *ob) if (use_newobj) { PyObject *cls; PyObject *newargtup; - int n, i; + Py_ssize_t n, i; /* Sanity checks. */ n = PyTuple_Size(argtup); @@ -2815,7 +2848,7 @@ Pickle_clear_memo(Picklerobject *self, PyObject *args) static PyObject * Pickle_getvalue(Picklerobject *self, PyObject *args) { - int l, i, rsize, ssize, clear=1, lm; + Py_ssize_t l, i, rsize, ssize, clear=1, lm; long ik; PyObject *k, *r; char *s, *p, *have_get; @@ -3314,7 +3347,7 @@ find_class(PyObject *py_module_name, PyObject *py_global_name, PyObject *fc) return global; } -static int +static Py_ssize_t marker(Unpicklerobject *self) { if (self->num_marks < 1) { @@ -3345,7 +3378,8 @@ load_int(Unpicklerobject *self) { PyObject *py_int = 0; char *endptr, *s; - int len, res = -1; + Py_ssize_t len; + int res = -1; long l; if ((len = self->readline_func(self, &s)) < 0) return -1; @@ -3477,7 +3511,8 @@ load_long(Unpicklerobject *self) { PyObject *l = 0; char *end, *s; - int len, res = -1; + Py_ssize_t len; + int res = -1; if ((len = self->readline_func(self, &s)) < 0) return -1; if (len < 2) return bad_readline(); @@ -3541,7 +3576,8 @@ load_float(Unpicklerobject *self) { PyObject *py_float = 0; char *endptr, *s; - int len, res = -1; + Py_ssize_t len; + int res = -1; double d; if ((len = self->readline_func(self, &s)) < 0) return -1; @@ -3597,7 +3633,8 @@ static int load_string(Unpicklerobject *self) { PyObject *str = 0; - int len, res = -1; + Py_ssize_t len; + int res = -1; char *s, *p; if ((len = self->readline_func(self, &s)) < 0) return -1; @@ -3639,7 +3676,7 @@ static int load_binstring(Unpicklerobject *self) { PyObject *py_string = 0; - long l; + Py_ssize_t l; char *s; if (self->read_func(self, &s, 4) < 0) return -1; @@ -3691,20 +3728,17 @@ static int load_unicode(Unpicklerobject *self) { PyObject *str = 0; - int len, res = -1; + Py_ssize_t len; char *s; if ((len = self->readline_func(self, &s)) < 0) return -1; if (len < 1) return bad_readline(); if (!( str = PyUnicode_DecodeRawUnicodeEscape(s, len - 1, NULL))) - goto finally; + return -1; PDATA_PUSH(self->stack, str, -1); return 0; - - finally: - return res; } #endif @@ -3714,7 +3748,7 @@ static int load_binunicode(Unpicklerobject *self) { PyObject *unicode; - long l; + Py_ssize_t l; char *s; if (self->read_func(self, &s, 4) < 0) return -1; @@ -3745,7 +3779,7 @@ static int load_tuple(Unpicklerobject *self) { PyObject *tup; - int i; + Py_ssize_t i; if ((i = marker(self)) < 0) return -1; if (!( tup=Pdata_popTuple(self->stack, i))) return -1; @@ -3798,7 +3832,7 @@ static int load_list(Unpicklerobject *self) { PyObject *list = 0; - int i; + Py_ssize_t i; if ((i = marker(self)) < 0) return -1; if (!( list=Pdata_popList(self->stack, i))) return -1; @@ -3810,7 +3844,7 @@ static int load_dict(Unpicklerobject *self) { PyObject *dict, *key, *value; - int i, j, k; + Py_ssize_t i, j, k; if ((i = marker(self)) < 0) return -1; j=self->stack->length; @@ -3886,7 +3920,7 @@ static int load_obj(Unpicklerobject *self) { PyObject *class, *tup, *obj=0; - int i; + Py_ssize_t i; if ((i = marker(self)) < 0) return -1; if (!( tup=Pdata_popTuple(self->stack, i+1))) return -1; @@ -3907,7 +3941,7 @@ static int load_inst(Unpicklerobject *self) { PyObject *tup, *class=0, *obj=0, *module_name, *class_name; - int i, len; + Py_ssize_t i, len; char *s; if ((i = marker(self)) < 0) return -1; @@ -3993,7 +4027,7 @@ static int load_global(Unpicklerobject *self) { PyObject *class = 0, *module_name = 0, *class_name = 0; - int len; + Py_ssize_t len; char *s; if ((len = self->readline_func(self, &s)) < 0) return -1; @@ -4024,7 +4058,7 @@ static int load_persid(Unpicklerobject *self) { PyObject *pid = 0; - int len; + Py_ssize_t len; char *s; if (self->pers_func) { @@ -4102,7 +4136,7 @@ load_binpersid(Unpicklerobject *self) static int load_pop(Unpicklerobject *self) { - int len = self->stack->length; + Py_ssize_t len = self->stack->length; /* Note that we split the (pickle.py) stack into two stacks, an object stack and a mark stack. We have to be clever and @@ -4127,7 +4161,7 @@ load_pop(Unpicklerobject *self) static int load_pop_mark(Unpicklerobject *self) { - int i; + Py_ssize_t i; if ((i = marker(self)) < 0) return -1; @@ -4142,7 +4176,7 @@ static int load_dup(Unpicklerobject *self) { PyObject *last; - int len; + Py_ssize_t len; if ((len = self->stack->length) <= 0) return stackUnderflow(); last=self->stack->data[len-1]; @@ -4156,7 +4190,7 @@ static int load_get(Unpicklerobject *self) { PyObject *py_str = 0, *value = 0; - int len; + Py_ssize_t len; char *s; int rc; @@ -4214,7 +4248,7 @@ load_long_binget(Unpicklerobject *self) PyObject *py_key = 0, *value = 0; unsigned char c; char *s; - long key; + Py_ssize_t key; int rc; if (self->read_func(self, &s, 4) < 0) return -1; @@ -4317,7 +4351,7 @@ static int load_put(Unpicklerobject *self) { PyObject *py_str = 0, *value = 0; - int len, l; + Py_ssize_t len, l; char *s; if ((l = self->readline_func(self, &s)) < 0) return -1; @@ -4337,7 +4371,7 @@ load_binput(Unpicklerobject *self) PyObject *py_key = 0, *value = 0; unsigned char key; char *s; - int len; + Py_ssize_t len; if (self->read_func(self, &s, 1) < 0) return -1; if (!( (len=self->stack->length) > 0 )) return stackUnderflow(); @@ -4356,10 +4390,10 @@ static int load_long_binput(Unpicklerobject *self) { PyObject *py_key = 0, *value = 0; - long key; + Py_ssize_t key; unsigned char c; char *s; - int len; + Py_ssize_t len; if (self->read_func(self, &s, 4) < 0) return -1; if (!( len=self->stack->length )) return stackUnderflow(); @@ -4382,10 +4416,10 @@ load_long_binput(Unpicklerobject *self) static int -do_append(Unpicklerobject *self, int x) +do_append(Unpicklerobject *self, Py_ssize_t x) { PyObject *value = 0, *list = 0, *append_method = 0; - int len, i; + Py_ssize_t len, i; len=self->stack->length; if (!( len >= x && x > 0 )) return stackUnderflow(); @@ -4451,11 +4485,11 @@ load_appends(Unpicklerobject *self) } -static int -do_setitems(Unpicklerobject *self, int x) +static Py_ssize_t +do_setitems(Unpicklerobject *self, Py_ssize_t x) { PyObject *value = 0, *key = 0, *dict = 0; - int len, i, r=0; + Py_ssize_t len, i, r=0; if (!( (len=self->stack->length) >= x && x > 0 )) return stackUnderflow(); @@ -4496,8 +4530,8 @@ load_build(Unpicklerobject *self) PyObject *state, *inst, *slotstate; PyObject *__setstate__; PyObject *d_key, *d_value; - Py_ssize_t i; int res = -1; + Py_ssize_t i; /* Stack is ... instance, state. We want to leave instance at * the stack top, possibly mutated via instance.__setstate__(state). @@ -4596,7 +4630,7 @@ load_build(Unpicklerobject *self) static int load_mark(Unpicklerobject *self) { - int s; + Py_ssize_t s; /* Note that we split the (pickle.py) stack into two stacks, an object stack and a mark stack. Here we push a mark onto the @@ -4604,14 +4638,14 @@ load_mark(Unpicklerobject *self) */ if ((self->num_marks + 1) >= self->marks_size) { - int *marks; + Py_ssize_t *marks; s=self->marks_size+20; if (s <= self->num_marks) s=self->num_marks + 1; if (self->marks == NULL) - marks=(int *)malloc(s * sizeof(int)); + marks=(Py_ssize_t *)malloc(s * sizeof(Py_ssize_t)); else - marks=(int *)realloc(self->marks, - s * sizeof(int)); + marks=(Py_ssize_t *)realloc(self->marks, + s * sizeof(Py_ssize_t)); if (!marks) { PyErr_NoMemory(); return -1; @@ -4981,7 +5015,7 @@ load(Unpicklerobject *self) static int noload_obj(Unpicklerobject *self) { - int i; + Py_ssize_t i; if ((i = marker(self)) < 0) return -1; return Pdata_clear(self->stack, i+1); @@ -4991,7 +5025,7 @@ noload_obj(Unpicklerobject *self) static int noload_inst(Unpicklerobject *self) { - int i; + Py_ssize_t i; char *s; if ((i = marker(self)) < 0) return -1; @@ -5068,7 +5102,7 @@ noload_append(Unpicklerobject *self) static int noload_appends(Unpicklerobject *self) { - int i; + Py_ssize_t i; if ((i = marker(self)) < 0) return -1; return Pdata_clear(self->stack, i); } @@ -5082,7 +5116,7 @@ noload_setitem(Unpicklerobject *self) static int noload_setitems(Unpicklerobject *self) { - int i; + Py_ssize_t i; if ((i = marker(self)) < 0) return -1; return Pdata_clear(self->stack, i); } diff --git a/Modules/cStringIO.c b/Modules/cStringIO.c index 9922d03..566851f 100644 --- a/Modules/cStringIO.c +++ b/Modules/cStringIO.c @@ -210,11 +210,8 @@ IO_creadline(PyObject *self, char **output) { if (n < end) n++; len = n - start; - if (len > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, - "length too large"); - return -1; - } + if (len > INT_MAX) + len = INT_MAX; *output=start; |