diff options
Diffstat (limited to 'Python/marshal.c')
-rw-r--r-- | Python/marshal.c | 365 |
1 files changed, 277 insertions, 88 deletions
diff --git a/Python/marshal.c b/Python/marshal.c index cc17329..727605a 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -1,8 +1,10 @@ /* Write Python objects to files and read them back. - This is intended for writing and reading compiled Python code only; - a true persistent storage facility would be much harder, since - it would have to take circular links and sharing into account. */ + This is primarily intended for writing and reading compiled Python code, + even though dicts, lists, sets and frozensets, not commonly seen in + code objects, are supported. + Version 3 of this protocol properly supports circular links + and sharing. */ #define PY_SSIZE_T_CLEAN @@ -31,16 +33,14 @@ #define TYPE_STOPITER 'S' #define TYPE_ELLIPSIS '.' #define TYPE_INT 'i' -/* TYPE_INT64 is deprecated. It is not - generated anymore, and support for reading it - will be removed in Python 3.4. */ -#define TYPE_INT64 'I' #define TYPE_FLOAT 'f' #define TYPE_BINARY_FLOAT 'g' #define TYPE_COMPLEX 'x' #define TYPE_BINARY_COMPLEX 'y' #define TYPE_LONG 'l' #define TYPE_STRING 's' +#define TYPE_INTERNED 't' +#define TYPE_REF 'r' #define TYPE_TUPLE '(' #define TYPE_LIST '[' #define TYPE_DICT '{' @@ -49,6 +49,7 @@ #define TYPE_UNKNOWN '?' #define TYPE_SET '<' #define TYPE_FROZENSET '>' +#define FLAG_REF '\x80' /* with a type, add obj to index */ #define WFERR_OK 0 #define WFERR_UNMARSHALLABLE 1 @@ -65,6 +66,7 @@ typedef struct { PyObject *current_filename; char *ptr; char *end; + PyObject *refs; /* dict on marshal, list on unmarshal */ int version; } WFILE; @@ -158,13 +160,17 @@ w_pstring(const char *s, Py_ssize_t n, WFILE *p) #endif #define PyLong_MARSHAL_RATIO (PyLong_SHIFT / PyLong_MARSHAL_SHIFT) +#define W_TYPE(t, p) do { \ + w_byte((t) | flag, (p)); \ +} while(0) + static void -w_PyLong(const PyLongObject *ob, WFILE *p) +w_PyLong(const PyLongObject *ob, char flag, WFILE *p) { Py_ssize_t i, j, n, l; digit d; - w_byte(TYPE_LONG, p); + W_TYPE(TYPE_LONG, p); if (Py_SIZE(ob) == 0) { w_long((long)0, p); return; @@ -201,10 +207,64 @@ w_PyLong(const PyLongObject *ob, WFILE *p) } while (d != 0); } +static int +w_ref(PyObject *v, char *flag, WFILE *p) +{ + PyObject *id; + PyObject *idx; + + if (p->version < 3 || p->refs == NULL) + return 0; /* not writing object references */ + + /* if it has only one reference, it definitely isn't shared */ + if (Py_REFCNT(v) == 1) + return 0; + + id = PyLong_FromVoidPtr((void*)v); + if (id == NULL) + goto err; + idx = PyDict_GetItem(p->refs, id); + if (idx != NULL) { + /* write the reference index to the stream */ + long w = PyLong_AsLong(idx); + Py_DECREF(id); + if (w == -1 && PyErr_Occurred()) { + goto err; + } + /* we don't store "long" indices in the dict */ + assert(0 <= w && w <= 0x7fffffff); + w_byte(TYPE_REF, p); + w_long(w, p); + return 1; + } else { + int ok; + Py_ssize_t s = PyDict_Size(p->refs); + /* we don't support long indices */ + if (s >= 0x7fffffff) { + PyErr_SetString(PyExc_ValueError, "too many objects"); + goto err; + } + idx = PyLong_FromSsize_t(s); + ok = idx && PyDict_SetItem(p->refs, id, idx) == 0; + Py_DECREF(id); + Py_XDECREF(idx); + if (!ok) + goto err; + *flag |= FLAG_REF; + return 0; + } +err: + p->error = WFERR_UNMARSHALLABLE; + return 1; +} + +static void +w_complex_object(PyObject *v, char flag, WFILE *p); + static void w_object(PyObject *v, WFILE *p) { - Py_ssize_t i, n; + char flag = '\0'; p->depth++; @@ -229,24 +289,35 @@ w_object(PyObject *v, WFILE *p) else if (v == Py_True) { w_byte(TYPE_TRUE, p); } - else if (PyLong_CheckExact(v)) { + else if (!w_ref(v, &flag, p)) + w_complex_object(v, flag, p); + + p->depth--; +} + +static void +w_complex_object(PyObject *v, char flag, WFILE *p) +{ + Py_ssize_t i, n; + + if (PyLong_CheckExact(v)) { long x = PyLong_AsLong(v); if ((x == -1) && PyErr_Occurred()) { PyLongObject *ob = (PyLongObject *)v; PyErr_Clear(); - w_PyLong(ob, p); + w_PyLong(ob, flag, p); } else { #if SIZEOF_LONG > 4 long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31); if (y && y != -1) { /* Too large for TYPE_INT */ - w_PyLong((PyLongObject*)v, p); + w_PyLong((PyLongObject*)v, flag, p); } else #endif { - w_byte(TYPE_INT, p); + W_TYPE(TYPE_INT, p); w_long(x, p); } } @@ -259,7 +330,7 @@ w_object(PyObject *v, WFILE *p) p->error = WFERR_UNMARSHALLABLE; return; } - w_byte(TYPE_BINARY_FLOAT, p); + W_TYPE(TYPE_BINARY_FLOAT, p); w_string((char*)buf, 8, p); } else { @@ -270,7 +341,7 @@ w_object(PyObject *v, WFILE *p) return; } n = strlen(buf); - w_byte(TYPE_FLOAT, p); + W_TYPE(TYPE_FLOAT, p); w_byte((int)n, p); w_string(buf, n, p); PyMem_Free(buf); @@ -284,7 +355,7 @@ w_object(PyObject *v, WFILE *p) p->error = WFERR_UNMARSHALLABLE; return; } - w_byte(TYPE_BINARY_COMPLEX, p); + W_TYPE(TYPE_BINARY_COMPLEX, p); w_string((char*)buf, 8, p); if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v), buf, 1) < 0) { @@ -295,7 +366,7 @@ w_object(PyObject *v, WFILE *p) } else { char *buf; - w_byte(TYPE_COMPLEX, p); + W_TYPE(TYPE_COMPLEX, p); buf = PyOS_double_to_string(PyComplex_RealAsDouble(v), 'g', 17, 0, NULL); if (!buf) { @@ -319,7 +390,7 @@ w_object(PyObject *v, WFILE *p) } } else if (PyBytes_CheckExact(v)) { - w_byte(TYPE_STRING, p); + W_TYPE(TYPE_STRING, p); w_pstring(PyBytes_AS_STRING(v), PyBytes_GET_SIZE(v), p); } else if (PyUnicode_CheckExact(v)) { @@ -330,12 +401,15 @@ w_object(PyObject *v, WFILE *p) p->error = WFERR_UNMARSHALLABLE; return; } - w_byte(TYPE_UNICODE, p); + if (p->version >= 3 && PyUnicode_CHECK_INTERNED(v)) + W_TYPE(TYPE_INTERNED, p); + else + W_TYPE(TYPE_UNICODE, p); w_pstring(PyBytes_AS_STRING(utf8), PyBytes_GET_SIZE(utf8), p); Py_DECREF(utf8); } else if (PyTuple_CheckExact(v)) { - w_byte(TYPE_TUPLE, p); + W_TYPE(TYPE_TUPLE, p); n = PyTuple_Size(v); W_SIZE(n, p); for (i = 0; i < n; i++) { @@ -343,7 +417,7 @@ w_object(PyObject *v, WFILE *p) } } else if (PyList_CheckExact(v)) { - w_byte(TYPE_LIST, p); + W_TYPE(TYPE_LIST, p); n = PyList_GET_SIZE(v); W_SIZE(n, p); for (i = 0; i < n; i++) { @@ -353,7 +427,7 @@ w_object(PyObject *v, WFILE *p) else if (PyDict_CheckExact(v)) { Py_ssize_t pos; PyObject *key, *value; - w_byte(TYPE_DICT, p); + W_TYPE(TYPE_DICT, p); /* This one is NULL object terminated! */ pos = 0; while (PyDict_Next(v, &pos, &key, &value)) { @@ -366,9 +440,9 @@ w_object(PyObject *v, WFILE *p) PyObject *value, *it; if (PyObject_TypeCheck(v, &PySet_Type)) - w_byte(TYPE_SET, p); + W_TYPE(TYPE_SET, p); else - w_byte(TYPE_FROZENSET, p); + W_TYPE(TYPE_FROZENSET, p); n = PyObject_Size(v); if (n == -1) { p->depth--; @@ -395,7 +469,7 @@ w_object(PyObject *v, WFILE *p) } else if (PyCode_Check(v)) { PyCodeObject *co = (PyCodeObject *)v; - w_byte(TYPE_CODE, p); + W_TYPE(TYPE_CODE, p); w_long(co->co_argcount, p); w_long(co->co_kwonlyargcount, p); w_long(co->co_nlocals, p); @@ -421,15 +495,14 @@ w_object(PyObject *v, WFILE *p) p->error = WFERR_UNMARSHALLABLE; return; } - w_byte(TYPE_STRING, p); + W_TYPE(TYPE_STRING, p); w_pstring(view.buf, view.len, p); PyBuffer_Release(&view); } else { - w_byte(TYPE_UNKNOWN, p); + W_TYPE(TYPE_UNKNOWN, p); p->error = WFERR_UNMARSHALLABLE; } - p->depth--; } /* version currently has no effect for writing ints. */ @@ -440,6 +513,7 @@ PyMarshal_WriteLongToFile(long x, FILE *fp, int version) wf.fp = fp; wf.error = WFERR_OK; wf.depth = 0; + wf.refs = NULL; wf.version = version; w_long(x, &wf); } @@ -451,8 +525,14 @@ PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version) wf.fp = fp; wf.error = WFERR_OK; wf.depth = 0; + if (version >= 3) { + if ((wf.refs = PyDict_New()) == NULL) + return; /* caller mush check PyErr_Occurred() */ + } else + wf.refs = NULL; wf.version = version; w_object(x, &wf); + Py_XDECREF(wf.refs); } typedef WFILE RFILE; /* Same struct with different invariants */ @@ -488,7 +568,7 @@ r_string(char *s, Py_ssize_t n, RFILE *p) data->ob_type->tp_name); } else { - read = PyBytes_GET_SIZE(data); + read = (int)PyBytes_GET_SIZE(data); if (read > 0) { if (read > n) { PyErr_Format(PyExc_ValueError, @@ -533,7 +613,7 @@ r_byte(RFILE *p) static int r_short(RFILE *p) { - register short x; + short x; unsigned char buffer[2]; r_string((char *) buffer, 2, p); @@ -547,7 +627,7 @@ r_short(RFILE *p) static long r_long(RFILE *p) { - register long x; + long x; unsigned char buffer[4]; r_string((char *) buffer, 4, p); @@ -562,42 +642,6 @@ r_long(RFILE *p) return x; } -/* r_long64 deals with the TYPE_INT64 code. On a machine with - sizeof(long) > 4, it returns a Python int object, else a Python long - object. Note that w_long64 writes out TYPE_INT if 32 bits is enough, - so there's no inefficiency here in returning a PyLong on 32-bit boxes - for everything written via TYPE_INT64 (i.e., if an int is written via - TYPE_INT64, it *needs* more than 32 bits). -*/ -static PyObject * -r_long64(RFILE *p) -{ - PyObject *result = NULL; - long lo4 = r_long(p); - long hi4 = r_long(p); - - if (!PyErr_Occurred()) { -#if SIZEOF_LONG > 4 - long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL); - result = PyLong_FromLong(x); -#else - unsigned char buf[8]; - int one = 1; - int is_little_endian = (int)*(char*)&one; - if (is_little_endian) { - memcpy(buf, &lo4, 4); - memcpy(buf+4, &hi4, 4); - } - else { - memcpy(buf, &hi4, 4); - memcpy(buf+4, &lo4, 4); - } - result = _PyLong_FromByteArray(buf, 8, is_little_endian, 1); -#endif - } - return result; -} - static PyObject * r_PyLong(RFILE *p) { @@ -667,6 +711,62 @@ r_PyLong(RFILE *p) return NULL; } +/* allocate the reflist index for a new object. Return -1 on failure */ +static Py_ssize_t +r_ref_reserve(int flag, RFILE *p) +{ + if (flag) { /* currently only FLAG_REF is defined */ + Py_ssize_t idx = PyList_Size(p->refs); + if (idx < 0) + return -1; + if (idx >= 0x7ffffffe) { + PyErr_SetString(PyExc_ValueError, "bad marshal data (index list too large)"); + return -1; + } + if (PyList_Append(p->refs, Py_None) < 0) + return -1; + return idx; + } else + return 0; +} + +/* insert the new object 'o' to the reflist at previously + * allocated index 'idx'. + * 'o' can be NULL, in which case nothing is done. + * if 'o' was non-NULL, and the function succeeds, 'o' is returned. + * if 'o' was non-NULL, and the function fails, 'o' is released and + * NULL returned. This simplifies error checking at the call site since + * a single test for NULL for the function result is enough. + */ +static PyObject * +r_ref_insert(PyObject *o, Py_ssize_t idx, int flag, RFILE *p) +{ + if (o != NULL && flag) { /* currently only FLAG_REF is defined */ + if (PyList_SetItem(p->refs, idx, o) < 0) { + Py_DECREF(o); /* release the new object */ + return NULL; + } else { + Py_INCREF(o); /* a reference for the list */ + } + } + return o; +} + +/* combination of both above, used when an object can be + * created whenever it is seen in the file, as opposed to + * after having loaded its sub-objects. + */ +static PyObject * +r_ref(PyObject *o, int flag, RFILE *p) +{ + if (o != NULL && flag) { /* currently only FLAG_REF is defined */ + if (PyList_Append(p->refs, o) < 0) { + Py_DECREF(o); /* release the new object */ + return NULL; + } + } + return o; +} static PyObject * r_object(RFILE *p) @@ -674,10 +774,18 @@ r_object(RFILE *p) /* NULL is a valid return value, it does not necessarily means that an exception is set. */ PyObject *v, *v2; + Py_ssize_t idx = 0; long i, n; - int type = r_byte(p); + int type, code = r_byte(p); + int flag; PyObject *retval; + if (code == EOF) { + PyErr_SetString(PyExc_EOFError, + "EOF read where object expected"); + return NULL; + } + p->depth++; if (p->depth > MAX_MARSHAL_STACK_DEPTH) { @@ -686,13 +794,15 @@ r_object(RFILE *p) return NULL; } - switch (type) { + flag = code & FLAG_REF; + type = code & ~FLAG_REF; - case EOF: - PyErr_SetString(PyExc_EOFError, - "EOF read where object expected"); - retval = NULL; - break; +#define R_REF(O) do{\ + if (flag) \ + O = r_ref(O, flag, p);\ +} while (0) + + switch (type) { case TYPE_NULL: retval = NULL; @@ -726,14 +836,12 @@ r_object(RFILE *p) case TYPE_INT: n = r_long(p); retval = PyErr_Occurred() ? NULL : PyLong_FromLong(n); - break; - - case TYPE_INT64: - retval = r_long64(p); + R_REF(retval); break; case TYPE_LONG: retval = r_PyLong(p); + R_REF(retval); break; case TYPE_FLOAT: @@ -754,6 +862,7 @@ r_object(RFILE *p) if (dx == -1.0 && PyErr_Occurred()) break; retval = PyFloat_FromDouble(dx); + R_REF(retval); break; } @@ -771,6 +880,7 @@ r_object(RFILE *p) break; } retval = PyFloat_FromDouble(x); + R_REF(retval); break; } @@ -804,6 +914,7 @@ r_object(RFILE *p) if (c.imag == -1.0 && PyErr_Occurred()) break; retval = PyComplex_FromCComplex(c); + R_REF(retval); break; } @@ -830,6 +941,7 @@ r_object(RFILE *p) break; } retval = PyComplex_FromCComplex(c); + R_REF(retval); break; } @@ -855,9 +967,11 @@ r_object(RFILE *p) break; } retval = v; + R_REF(retval); break; case TYPE_UNICODE: + case TYPE_INTERNED: { char *buffer; @@ -871,19 +985,31 @@ r_object(RFILE *p) retval = NULL; break; } - buffer = PyMem_NEW(char, n); - if (buffer == NULL) { - retval = PyErr_NoMemory(); - break; - } - if (r_string(buffer, n, p) != n) { + if (n != 0) { + buffer = PyMem_NEW(char, n); + if (buffer == NULL) { + retval = PyErr_NoMemory(); + break; + } + if (r_string(buffer, n, p) != n) { + PyMem_DEL(buffer); + retval = NULL; + break; + } + v = PyUnicode_DecodeUTF8(buffer, n, "surrogatepass"); PyMem_DEL(buffer); + } + else { + v = PyUnicode_New(0, 0); + } + if (v == NULL) { retval = NULL; break; } - v = PyUnicode_DecodeUTF8(buffer, n, "surrogatepass"); - PyMem_DEL(buffer); + if (type == TYPE_INTERNED) + PyUnicode_InternInPlace(&v); retval = v; + R_REF(retval); break; } @@ -899,6 +1025,7 @@ r_object(RFILE *p) break; } v = PyTuple_New(n); + R_REF(v); if (v == NULL) { retval = NULL; break; @@ -930,6 +1057,7 @@ r_object(RFILE *p) break; } v = PyList_New(n); + R_REF(v); if (v == NULL) { retval = NULL; break; @@ -951,6 +1079,7 @@ r_object(RFILE *p) case TYPE_DICT: v = PyDict_New(); + R_REF(v); if (v == NULL) { retval = NULL; break; @@ -986,6 +1115,16 @@ r_object(RFILE *p) break; } v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL); + if (type == TYPE_SET) { + R_REF(v); + } else { + /* must use delayed registration of frozensets because they must + * be init with a refcount of 1 + */ + idx = r_ref_reserve(flag, p); + if (idx < 0) + Py_CLEAR(v); /* signal error */ + } if (v == NULL) { retval = NULL; break; @@ -1008,6 +1147,8 @@ r_object(RFILE *p) } Py_DECREF(v2); } + if (type != TYPE_SET) + v = r_ref_insert(v, idx, flag, p); retval = v; break; @@ -1029,6 +1170,12 @@ r_object(RFILE *p) int firstlineno; PyObject *lnotab = NULL; + idx = r_ref_reserve(flag, p); + if (idx < 0) { + retval = NULL; + break; + } + v = NULL; /* XXX ignore long->int overflows for now */ @@ -1094,6 +1241,7 @@ r_object(RFILE *p) code, consts, names, varnames, freevars, cellvars, filename, name, firstlineno, lnotab); + v = r_ref_insert(v, idx, flag, p); code_error: Py_XDECREF(code); @@ -1109,6 +1257,23 @@ r_object(RFILE *p) retval = v; break; + case TYPE_REF: + n = r_long(p); + if (n < 0 || n >= PyList_GET_SIZE(p->refs)) { + PyErr_SetString(PyExc_ValueError, "bad marshal data (invalid reference)"); + retval = NULL; + break; + } + v = PyList_GET_ITEM(p->refs, n); + if (v == Py_None) { + PyErr_SetString(PyExc_ValueError, "bad marshal data (invalid reference)"); + retval = NULL; + break; + } + Py_INCREF(v); + retval = v; + break; + default: /* Bogus data got written, which isn't ideal. This will let you keep working and recover. */ @@ -1214,7 +1379,11 @@ PyMarshal_ReadObjectFromFile(FILE *fp) rf.current_filename = NULL; rf.depth = 0; rf.ptr = rf.end = NULL; + rf.refs = PyList_New(0); + if (rf.refs == NULL) + return NULL; result = r_object(&rf); + Py_DECREF(rf.refs); return result; } @@ -1229,7 +1398,11 @@ PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len) rf.ptr = str; rf.end = str + len; rf.depth = 0; + rf.refs = PyList_New(0); + if (rf.refs == NULL) + return NULL; result = r_object(&rf); + Py_DECREF(rf.refs); return result; } @@ -1248,7 +1421,13 @@ PyMarshal_WriteObjectToString(PyObject *x, int version) wf.error = WFERR_OK; wf.depth = 0; wf.version = version; + if (version >= 3) { + if ((wf.refs = PyDict_New()) == NULL) + return NULL; + } else + wf.refs = NULL; w_object(x, &wf); + Py_XDECREF(wf.refs); if (wf.str != NULL) { char *base = PyBytes_AS_STRING((PyBytesObject *)wf.str); if (wf.ptr - base > PY_SSIZE_T_MAX) { @@ -1320,6 +1499,8 @@ marshal_load(PyObject *self, PyObject *f) * Make a call to the read method, but read zero bytes. * This is to ensure that the object passed in at least * has a read method which returns bytes. + * This can be removed if we guarantee good error handling + * for r_string() */ data = _PyObject_CallMethodId(f, &PyId_read, "i", 0); if (data == NULL) @@ -1335,7 +1516,11 @@ marshal_load(PyObject *self, PyObject *f) rf.fp = NULL; rf.readable = f; rf.current_filename = NULL; - result = read_object(&rf); + if ((rf.refs = PyList_New(0)) != NULL) { + result = read_object(&rf); + Py_DECREF(rf.refs); + } else + result = NULL; } Py_DECREF(data); return result; @@ -1392,8 +1577,11 @@ marshal_loads(PyObject *self, PyObject *args) rf.ptr = s; rf.end = s + n; rf.depth = 0; + if ((rf.refs = PyList_New(0)) == NULL) + return NULL; result = read_object(&rf); PyBuffer_Release(&p); + Py_DECREF(rf.refs); return result; } @@ -1433,6 +1621,7 @@ Variables:\n\ version -- indicates the format that the module uses. Version 0 is the\n\ historical format, version 1 shares interned strings and version 2\n\ uses a binary format for floating point numbers.\n\ + Version 3 shares common object references (New in version 3.4).\n\ \n\ Functions:\n\ \n\ |