From b99f762f10edb2646a634c2290ecb064bd52e5c7 Mon Sep 17 00:00:00 2001 From: "Travis E. Oliphant" Date: Sat, 18 Aug 2007 11:21:56 +0000 Subject: Merged in py3k-buffer branch to main line. All objects now use the buffer protocol in PEP 3118. --- Include/Python.h | 1 + Include/abstract.h | 110 ++++++++++ Include/bytesobject.h | 1 + Include/memoryobject.h | 67 ++++++ Include/object.h | 67 +++++- Include/pyerrors.h | 1 + Makefile.pre.in | 2 + Modules/_ctypes/_ctypes.c | 39 ++-- Modules/_sre.c | 44 ++-- Modules/arraymodule.c | 117 ++++++---- Modules/mmapmodule.c | 73 +++---- Objects/abstract.c | 463 ++++++++++++++++++++++++++++++++++----- Objects/bufferobject.c | 445 +++++++++++++++----------------------- Objects/bytesobject.c | 257 +++++++++++++--------- Objects/exceptions.c | 5 + Objects/memoryobject.c | 540 ++++++++++++++++++++++++++++++++++++++++++++++ Objects/stringobject.c | 47 +--- Objects/typeobject.c | 6 +- Objects/unicodeobject.c | 63 ++---- Python/bltinmodule.c | 1 + Python/getargs.c | 55 +++-- Python/marshal.c | 12 +- 22 files changed, 1730 insertions(+), 686 deletions(-) create mode 100644 Include/memoryobject.h create mode 100644 Objects/memoryobject.c diff --git a/Include/Python.h b/Include/Python.h index 43f16f7..32b2aa5 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -77,6 +77,7 @@ #include "rangeobject.h" #include "stringobject.h" #include "bufferobject.h" +#include "memoryobject.h" #include "tupleobject.h" #include "listobject.h" #include "dictobject.h" diff --git a/Include/abstract.h b/Include/abstract.h index 22b5d01..b8cc59c 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -475,6 +475,12 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/ This is the equivalent of the Python statement: del o[key]. */ + /* old buffer API + FIXME: usage of these should all be replaced in Python itself + but for backwards compatibility we will implement them. + Their usage without a corresponding "unlock" mechansim + may create issues (but they would already be there). */ + PyAPI_FUNC(int) PyObject_AsCharBuffer(PyObject *obj, const char **buffer, Py_ssize_t *buffer_len); @@ -527,6 +533,110 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/ an exception set. */ + /* new buffer API */ + +#define PyObject_CheckBuffer(obj) \ + (((obj)->ob_type->tp_as_buffer != NULL) && \ + ((obj)->ob_type->tp_as_buffer->bf_getbuffer != NULL)) + + /* Return 1 if the getbuffer function is available, otherwise + return 0 */ + + PyAPI_FUNC(int) PyObject_GetBuffer(PyObject *obj, PyBuffer *view, + int flags); + + /* This is a C-API version of the getbuffer function call. It checks + to make sure object has the required function pointer and issues the + call. Returns -1 and raises an error on failure and returns 0 on + success + */ + + + PyAPI_FUNC(void) PyObject_ReleaseBuffer(PyObject *obj, PyBuffer *view); + + + /* C-API version of the releasebuffer function call. It + checks to make sure the object has the required function + pointer and issues the call. The obj must have the buffer + interface or this function will cause a segfault (i.e. it + is assumed to be called only after a corresponding + getbuffer which already verified the existence of the + tp_as_buffer pointer). + + Returns 0 on success and -1 (with an error raised) on + failure. This function always succeeds (as a NO-OP) if + there is no releasebuffer function for the object so that + it can always be called when the consumer is done with the + buffer + */ + + PyAPI_FUNC(void *) PyBuffer_GetPointer(PyBuffer *view, Py_ssize_t *indices); + + /* Get the memory area pointed to by the indices for the buffer given. + Note that view->ndim is the assumed size of indices + */ + + PyAPI_FUNC(int) PyBuffer_SizeFromFormat(const char *); + + /* Return the implied itemsize of the data-format area from a + struct-style description */ + + + + PyAPI_FUNC(int) PyBuffer_ToContiguous(void *buf, PyBuffer *view, + Py_ssize_t len, char fort); + + PyAPI_FUNC(int) PyBuffer_FromContiguous(PyBuffer *view, void *buf, + Py_ssize_t len, char fort); + + + /* Copy len bytes of data from the contiguous chunk of memory + pointed to by buf into the buffer exported by obj. Return + 0 on success and return -1 and raise a PyBuffer_Error on + error (i.e. the object does not have a buffer interface or + it is not working). + + If fortran is 'F', then if the object is multi-dimensional, + then the data will be copied into the array in + Fortran-style (first dimension varies the fastest). If + fortran is 'C', then the data will be copied into the array + in C-style (last dimension varies the fastest). If fortran + is 'A', then it does not matter and the copy will be made + in whatever way is more efficient. + + */ + + PyAPI_FUNC(int) PyObject_CopyData(PyObject *dest, PyObject *src); + + /* Copy the data from the src buffer to the buffer of destination + */ + + PyAPI_FUNC(int) PyBuffer_IsContiguous(PyBuffer *view, char fortran); + + + PyAPI_FUNC(void) PyBuffer_FillContiguousStrides(int ndims, + Py_ssize_t *shape, + Py_ssize_t *strides, + int itemsize, + char fort); + + /* Fill the strides array with byte-strides of a contiguous + (Fortran-style if fortran is 'F' or C-style otherwise) + array of the given shape with the given number of bytes + per element. + */ + + PyAPI_FUNC(int) PyBuffer_FillInfo(PyBuffer *view, void *buf, + Py_ssize_t len, int readonly, + int flags); + + /* Fills in a buffer-info structure correctly for an exporter + that can only share a contiguous chunk of memory of + "unsigned bytes" of the given length. Returns 0 on success + and -1 (with raising an error) on error. + */ + + /* Iterators */ PyAPI_FUNC(PyObject *) PyObject_GetIter(PyObject *); diff --git a/Include/bytesobject.h b/Include/bytesobject.h index f1e01d1..12ecf64 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -21,6 +21,7 @@ extern "C" { /* Object layout */ typedef struct { PyObject_VAR_HEAD + int ob_exports; /* how many buffer exports */ Py_ssize_t ob_alloc; /* How many bytes allocated */ char *ob_bytes; } PyBytesObject; diff --git a/Include/memoryobject.h b/Include/memoryobject.h new file mode 100644 index 0000000..eb49c2e --- /dev/null +++ b/Include/memoryobject.h @@ -0,0 +1,67 @@ + +/* Memory object interface */ + +#ifndef Py_MEMORYOBJECT_H +#define Py_MEMORYOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + PyObject_HEAD + PyObject *base; + PyBuffer view; +} PyMemoryViewObject; + + +PyAPI_DATA(PyTypeObject) PyMemoryView_Type; + +#define PyMemory_Check(op) (Py_Type(op) == &PyMemoryView_Type) +#define PyMemoryView(op) (((PyMemoryViewObject *)(op))->view) + +#define Py_END_OF_MEMORY (-1) + +PyAPI_FUNC(PyObject *) PyMemoryView_GetContiguous(PyObject *base, int buffertype, + char fort); + + /* Return a contiguous chunk of memory representing the buffer + from an object in a memory view object. If a copy is made then the + base object for the memory view will be a *new* bytes object. + + Otherwise, the base-object will be the object itself and no + data-copying will be done. + + The buffertype argument can be PyBUF_READ, PyBUF_WRITE, + PyBUF_UPDATEIFCOPY to determine whether the returned buffer + should be READONLY, WRITEABLE, or set to update the + original buffer if a copy must be made. If buffertype is + PyBUF_WRITE and the buffer is not contiguous an error will + be raised. In this circumstance, the user can use + PyBUF_UPDATEIFCOPY to ensure that a a writeable temporary + contiguous buffer is returned. The contents of this + contiguous buffer will be copied back into the original + object after the memoryview object is deleted as long as + the original object is writeable and allows setting its + memory to "readonly". If this is not allowed by the + original object, then a BufferError is raised. + + If the object is multi-dimensional and if fortran is 'F', + the first dimension of the underlying array will vary the + fastest in the buffer. If fortran is 'C', then the last + dimension will vary the fastest (C-style contiguous). If + fortran is 'A', then it does not matter and you will get + whatever the object decides is more efficient. + + A new reference is returned that must be DECREF'd when finished. + */ + +PyAPI_FUNC(PyObject *) PyMemoryView_FromObject(PyObject *base); + +PyAPI_FUNC(PyObject *) PyMemoryView_FromMemory(PyBuffer *info); + /* create new if bufptr is NULL + will be a new bytesobject in base */ + +#ifdef __cplusplus +} +#endif +#endif /* !Py_MEMORYOBJECT_H */ diff --git a/Include/object.h b/Include/object.h index 9470ed7..d936bca 100644 --- a/Include/object.h +++ b/Include/object.h @@ -140,11 +140,59 @@ typedef int(*ssizeobjargproc)(PyObject *, Py_ssize_t, PyObject *); typedef int(*ssizessizeobjargproc)(PyObject *, Py_ssize_t, Py_ssize_t, PyObject *); typedef int(*objobjargproc)(PyObject *, PyObject *, PyObject *); -/* ssize_t-based buffer interface */ -typedef Py_ssize_t (*readbufferproc)(PyObject *, Py_ssize_t, void **); -typedef Py_ssize_t (*writebufferproc)(PyObject *, Py_ssize_t, void **); -typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *); -typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **); + +/* buffer interface */ +typedef struct bufferinfo { + void *buf; + Py_ssize_t len; + Py_ssize_t itemsize; + int readonly; + int ndim; + char *format; + Py_ssize_t *shape; + Py_ssize_t *strides; + Py_ssize_t *suboffsets; + void *internal; +} PyBuffer; + +typedef int (*getbufferproc)(PyObject *, PyBuffer *, int); +typedef void (*releasebufferproc)(PyObject *, PyBuffer *); + + /* Flags for getting buffers */ +#define PyBUF_SIMPLE 0 +#define PyBUF_CHARACTER 1 +#define PyBUF_WRITEABLE 0x0002 +#define PyBUF_LOCKDATA 0x0004 +#define PyBUF_FORMAT 0x0008 +#define PyBUF_ND 0x0010 +#define PyBUF_STRIDES (0x0020 | PyBUF_ND) +#define PyBUF_C_CONTIGUOUS (0x0040 | PyBUF_STRIDES) +#define PyBUF_F_CONTIGUOUS (0x0080 | PyBUF_STRIDES) +#define PyBUF_ANY_CONTIGUOUS (0x0100 | PyBUF_STRIDES) +#define PyBUF_INDIRECT (0x0200 | PyBUF_STRIDES) + +#define PyBUF_CONTIG (PyBUF_ND | PyBUF_WRITEABLE) +#define PyBUF_CONTIG_RO (PyBUF_ND) +#define PyBUF_CONTIG_LCK (PyBUF_ND | PyBUF_LOCKDATA) + +#define PyBUF_STRIDED (PyBUF_STRIDES | PyBUF_WRITEABLE) +#define PyBUF_STRIDED_RO (PyBUF_STRIDES) +#define PyBUF_STRIDED_LCK (PyBUF_STRIDES | PyBUF_LOCKDATA) + +#define PyBUF_RECORDS (PyBUF_STRIDES | PyBUF_WRITEABLE | PyBUF_FORMAT) +#define PyBUF_RECORDS_RO (PyBUF_STRIDES | PyBUF_FORMAT) +#define PyBUF_RECORDS_LCK (PyBUF_STRIDES | PyBUF_LOCKDATA | PyBUF_FORMAT) + +#define PyBUF_FULL (PyBUF_INDIRECT | PyBUF_WRITEABLE | PyBUF_FORMAT) +#define PyBUF_FULL_RO (PyBUF_INDIRECT | PyBUF_FORMAT) +#define PyBUF_FULL_LCK (PyBUF_INDIRECT | PyBUF_LOCKDATA | PyBUF_FORMAT) + + +#define PyBUF_READ 0x100 +#define PyBUF_WRITE 0x200 +#define PyBUF_SHADOW 0x400 + +/* End buffer interface */ typedef int (*objobjproc)(PyObject *, PyObject *); typedef int (*visitproc)(PyObject *, void *); @@ -218,14 +266,13 @@ typedef struct { objobjargproc mp_ass_subscript; } PyMappingMethods; + typedef struct { - readbufferproc bf_getreadbuffer; - writebufferproc bf_getwritebuffer; - segcountproc bf_getsegcount; - charbufferproc bf_getcharbuffer; + getbufferproc bf_getbuffer; + releasebufferproc bf_releasebuffer; + inquiry bf_multisegment; } PyBufferProcs; - typedef void (*freefunc)(void *); typedef void (*destructor)(PyObject *); typedef int (*printfunc)(PyObject *, FILE *, int); diff --git a/Include/pyerrors.h b/Include/pyerrors.h index b1cc429..c519cb6 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -138,6 +138,7 @@ PyAPI_DATA(PyObject *) PyExc_UnicodeDecodeError; PyAPI_DATA(PyObject *) PyExc_UnicodeTranslateError; PyAPI_DATA(PyObject *) PyExc_ValueError; PyAPI_DATA(PyObject *) PyExc_ZeroDivisionError; +PyAPI_DATA(PyObject *) PyExc_BufferError; #ifdef MS_WINDOWS PyAPI_DATA(PyObject *) PyExc_WindowsError; #endif diff --git a/Makefile.pre.in b/Makefile.pre.in index 51e9a90..4e9bab1 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -300,6 +300,7 @@ OBJECT_OBJS= \ Objects/listobject.o \ Objects/longobject.o \ Objects/dictobject.o \ + Objects/memoryobject.o \ Objects/methodobject.o \ Objects/moduleobject.o \ Objects/object.o \ @@ -533,6 +534,7 @@ PYTHON_HEADERS= \ Include/iterobject.h \ Include/listobject.h \ Include/longobject.h \ + Include/memoryobject.h \ Include/methodobject.h \ Include/modsupport.h \ Include/moduleobject.h \ diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index e6db969..fcb9085 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -739,22 +739,33 @@ CharArray_set_raw(CDataObject *self, PyObject *value) { char *ptr; Py_ssize_t size; + int rel = 0; + PyBuffer view; + if (PyBuffer_Check(value)) { - size = Py_Type(value)->tp_as_buffer->bf_getreadbuffer(value, 0, (void *)&ptr); - if (size < 0) - return -1; + if (PyObject_GetBuffer(value, &view, PyBUF_SIMPLE) < 0) + return -1; + size = view.len; + ptr = view.buf; + rel = 1; } else if (-1 == PyString_AsStringAndSize(value, &ptr, &size)) { return -1; } if (size > self->b_size) { PyErr_SetString(PyExc_ValueError, "string too long"); - return -1; + goto fail; } memcpy(self->b_ptr, ptr, size); + if (rel) + PyObject_ReleaseBuffer(value, &view); return 0; + fail: + if (rel) + PyObject_ReleaseBuffer(value, &view); + return -1; } static PyObject * @@ -2072,29 +2083,15 @@ static PyMemberDef CData_members[] = { { NULL }, }; -static Py_ssize_t CData_GetBuffer(PyObject *_self, Py_ssize_t seg, void **pptr) +static int CData_GetBuffer(PyObject *_self, PyBuffer *view, int flags) { CDataObject *self = (CDataObject *)_self; - if (seg != 0) { - /* Hm. Must this set an exception? */ - return -1; - } - *pptr = self->b_ptr; - return self->b_size; -} - -static Py_ssize_t CData_GetSegcount(PyObject *_self, Py_ssize_t *lenp) -{ - if (lenp) - *lenp = 1; - return 1; + return PyBuffer_FillInfo(view, self->b_ptr, self->b_size, 0, flags); } static PyBufferProcs CData_as_buffer = { CData_GetBuffer, - CData_GetBuffer, - CData_GetSegcount, - NULL, + NULL, }; /* diff --git a/Modules/_sre.c b/Modules/_sre.c index 012f127..295bc93 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -1672,44 +1672,38 @@ getstring(PyObject* string, Py_ssize_t* p_length, int* p_charsize) Py_ssize_t size, bytes; int charsize; void* ptr; - -#if defined(HAVE_UNICODE) - if (PyUnicode_Check(string)) { - /* unicode strings doesn't always support the buffer interface */ - ptr = (void*) PyUnicode_AS_DATA(string); - bytes = PyUnicode_GET_DATA_SIZE(string); - size = PyUnicode_GET_SIZE(string); - charsize = sizeof(Py_UNICODE); - - } else { -#endif + PyBuffer view; /* get pointer to string buffer */ + view.len = -1; buffer = Py_Type(string)->tp_as_buffer; - if (!buffer || !buffer->bf_getreadbuffer || !buffer->bf_getsegcount || - buffer->bf_getsegcount(string, NULL) != 1) { - PyErr_SetString(PyExc_TypeError, "expected string or buffer"); - return NULL; + if (!buffer || !buffer->bf_getbuffer || + (*buffer->bf_getbuffer)(string, &view, PyBUF_SIMPLE) < 0) { + PyErr_SetString(PyExc_TypeError, "expected string or buffer"); + return NULL; } /* determine buffer size */ - bytes = buffer->bf_getreadbuffer(string, 0, &ptr); + bytes = view.len; + ptr = view.buf; + + /* Release the buffer immediately --- possibly dangerous + but doing something else would require some re-factoring + */ + PyObject_ReleaseBuffer(string, &view); + if (bytes < 0) { PyErr_SetString(PyExc_TypeError, "buffer has negative size"); return NULL; } /* determine character size */ -#if PY_VERSION_HEX >= 0x01060000 size = PyObject_Size(string); -#else - size = PyObject_Length(string); -#endif if (PyString_Check(string) || bytes == size) charsize = 1; #if defined(HAVE_UNICODE) - else if (bytes == (Py_ssize_t) (size * sizeof(Py_UNICODE))) + else if (bytes == (Py_ssize_t) (size * sizeof(Py_UNICODE))) charsize = sizeof(Py_UNICODE); #endif else { @@ -1717,13 +1711,13 @@ getstring(PyObject* string, Py_ssize_t* p_length, int* p_charsize) return NULL; } -#if defined(HAVE_UNICODE) - } -#endif - *p_length = size; *p_charsize = charsize; + if (ptr == NULL) { + PyErr_SetString(PyExc_ValueError, + "Buffer is NULL"); + } return ptr; } diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 8e8dce2..656fc09 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -26,6 +26,7 @@ struct arraydescr { int itemsize; PyObject * (*getitem)(struct arrayobject *, Py_ssize_t); int (*setitem)(struct arrayobject *, Py_ssize_t, PyObject *); + const char *formats; }; typedef struct arrayobject { @@ -34,10 +35,19 @@ typedef struct arrayobject { Py_ssize_t allocated; struct arraydescr *ob_descr; PyObject *weakreflist; /* List of weak references */ + int ob_exports; /* Number of exported buffers */ } arrayobject; static PyTypeObject Arraytype; +#ifdef Py_UNICODE_WIDE +#define PyArr_UNI 'w' +/*static const char *PyArr_UNISTR = "w"; */ +#else +#define PyArr_UNI 'u' +/*static const char *PyArr_UNISTR = "u"; */ +#endif + #define array_Check(op) PyObject_TypeCheck(op, &Arraytype) #define array_CheckExact(op) (Py_Type(op) == &Arraytype) @@ -59,6 +69,12 @@ array_resize(arrayobject *self, Py_ssize_t newsize) return 0; } + if (self->ob_exports > 0) { + PyErr_SetString(PyExc_BufferError, + "cannot resize an array that is exporting data"); + return -1; + } + /* This over-allocates proportional to the array size, making room * for additional growth. The over-allocation is mild, but is * enough to give linear-time amortized behavior over a long @@ -370,11 +386,12 @@ d_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) return 0; } + /* Description of types */ static struct arraydescr descriptors[] = { {'b', sizeof(char), b_getitem, b_setitem}, {'B', sizeof(char), BB_getitem, BB_setitem}, - {'u', sizeof(Py_UNICODE), u_getitem, u_setitem}, + {PyArr_UNI, sizeof(Py_UNICODE), u_getitem, u_setitem}, {'h', sizeof(short), h_getitem, h_setitem}, {'H', sizeof(short), HH_getitem, HH_setitem}, {'i', sizeof(int), i_getitem, i_setitem}, @@ -424,6 +441,7 @@ newarrayobject(PyTypeObject *type, Py_ssize_t size, struct arraydescr *descr) op->ob_descr = descr; op->allocated = size; op->weakreflist = NULL; + op->ob_exports = 0; return (PyObject *) op; } @@ -1403,10 +1421,10 @@ array_fromunicode(arrayobject *self, PyObject *args) if (!PyArg_ParseTuple(args, "u#:fromunicode", &ustr, &n)) return NULL; - if (self->ob_descr->typecode != 'u') { + if (self->ob_descr->typecode != PyArr_UNI) { PyErr_SetString(PyExc_ValueError, "fromunicode() may only be called on " - "type 'u' arrays"); + "unicode type arrays"); return NULL; } if (n > 0) { @@ -1431,7 +1449,7 @@ PyDoc_STRVAR(fromunicode_doc, "fromunicode(ustr)\n\ \n\ Extends this array with data from the unicode string ustr.\n\ -The array must be a type 'u' array; otherwise a ValueError\n\ +The array must be a unicode type array; otherwise a ValueError\n\ is raised. Use array.fromstring(ustr.decode(...)) to\n\ append Unicode data to an array of some other type."); @@ -1439,9 +1457,9 @@ append Unicode data to an array of some other type."); static PyObject * array_tounicode(arrayobject *self, PyObject *unused) { - if (self->ob_descr->typecode != 'u') { + if (self->ob_descr->typecode != PyArr_UNI) { PyErr_SetString(PyExc_ValueError, - "tounicode() may only be called on type 'u' arrays"); + "tounicode() may only be called on unicode type arrays"); return NULL; } return PyUnicode_FromUnicode((Py_UNICODE *) self->ob_item, Py_Size(self)); @@ -1451,7 +1469,7 @@ PyDoc_STRVAR(tounicode_doc, "tounicode() -> unicode\n\ \n\ Convert the array to a unicode string. The array must be\n\ -a type 'u' array; otherwise a ValueError is raised. Use\n\ +a unicode type array; otherwise a ValueError is raised. Use\n\ array.tostring().decode() to obtain a unicode string from\n\ an array of some other type."); @@ -1542,7 +1560,7 @@ array_repr(arrayobject *a) if (len == 0) { return PyUnicode_FromFormat("array('%c')", typecode); } - if (typecode == 'u') + if (typecode == PyArr_UNI) v = array_tounicode(a, NULL); else v = array_tolist(a, NULL); @@ -1720,40 +1738,56 @@ static PyMappingMethods array_as_mapping = { static const void *emptybuf = ""; -static Py_ssize_t -array_buffer_getreadbuf(arrayobject *self, Py_ssize_t index, const void **ptr) -{ - if ( index != 0 ) { - PyErr_SetString(PyExc_SystemError, - "Accessing non-existent array segment"); - return -1; - } - *ptr = (void *)self->ob_item; - if (*ptr == NULL) - *ptr = emptybuf; - return Py_Size(self)*self->ob_descr->itemsize; -} -static Py_ssize_t -array_buffer_getwritebuf(arrayobject *self, Py_ssize_t index, const void **ptr) +static int +array_buffer_getbuf(arrayobject *self, PyBuffer *view, int flags) { - if ( index != 0 ) { - PyErr_SetString(PyExc_SystemError, - "Accessing non-existent array segment"); - return -1; - } - *ptr = (void *)self->ob_item; - if (*ptr == NULL) - *ptr = emptybuf; - return Py_Size(self)*self->ob_descr->itemsize; + if ((flags & PyBUF_CHARACTER)) { + PyErr_SetString(PyExc_TypeError, + "Cannot be a character buffer"); + return -1; + } + if ((flags & PyBUF_LOCKDATA)) { + PyErr_SetString(PyExc_BufferError, + "Cannot lock data"); + return -1; + } + if (view==NULL) goto finish; + + view->buf = (void *)self->ob_item; + if (view->buf == NULL) + view->buf = (void *)emptybuf; + view->len = (Py_Size(self)) * self->ob_descr->itemsize; + view->readonly = 0; + view->ndim = 1; + view->itemsize = self->ob_descr->itemsize; + view->suboffsets = NULL; + view->shape = NULL; + if ((flags & PyBUF_ND)==PyBUF_ND) { + view->shape = &((Py_Size(self))); + } + view->strides = NULL; + if ((flags & PyBUF_STRIDES)==PyBUF_STRIDES) + view->strides = &(view->itemsize); + view->format = NULL; + view->internal = NULL; + if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) { + view->internal = malloc(3); + view->format = view->internal; + view->format[0] = (char)(self->ob_descr->typecode); + view->format[1] = '\0'; + } + + finish: + self->ob_exports++; + return 0; } -static Py_ssize_t -array_buffer_getsegcount(arrayobject *self, Py_ssize_t *lenp) +static void +array_buffer_relbuf(arrayobject *self, PyBuffer *view) { - if ( lenp ) - *lenp = Py_Size(self)*self->ob_descr->itemsize; - return 1; + free(view->internal); + self->ob_exports--; } static PySequenceMethods array_as_sequence = { @@ -1770,10 +1804,8 @@ static PySequenceMethods array_as_sequence = { }; static PyBufferProcs array_as_buffer = { - (readbufferproc)array_buffer_getreadbuf, - (writebufferproc)array_buffer_getwritebuf, - (segcountproc)array_buffer_getsegcount, - NULL, + (getbufferproc)array_buffer_getbuf, + (releasebufferproc)array_buffer_relbuf }; static PyObject * @@ -1792,7 +1824,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (!(initial == NULL || PyList_Check(initial) || PyBytes_Check(initial) || PyString_Check(initial) || PyTuple_Check(initial) - || (c == 'u' && PyUnicode_Check(initial)))) { + || (c == PyArr_UNI && PyUnicode_Check(initial)))) { it = PyObject_GetIter(initial); if (it == NULL) return NULL; @@ -1900,6 +1932,7 @@ is a single character. The following type codes are defined:\n\ 'H' unsigned integer 2 \n\ 'i' signed integer 2 \n\ 'I' unsigned integer 2 \n\ + 'w' unicode character 4 \n\ 'l' signed integer 4 \n\ 'L' unsigned integer 4 \n\ 'f' floating point 4 \n\ diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 1626d72..16a9dd2 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -75,6 +75,7 @@ typedef struct { char * data; size_t size; size_t pos; + int exports; #ifdef MS_WINDOWS HANDLE map_handle; @@ -119,6 +120,11 @@ mmap_object_dealloc(mmap_object *m_obj) static PyObject * mmap_close_method(mmap_object *self, PyObject *unused) { + if (self->exports > 0) { + PyErr_SetString(PyExc_BufferError, "cannot close "\ + "exported pointers exist"); + return NULL; + } #ifdef MS_WINDOWS /* For each resource we maintain, we need to check the value is valid, and if so, free the resource @@ -277,8 +283,13 @@ is_writeable(mmap_object *self) static int is_resizeable(mmap_object *self) { + if (self->exports > 0) { + PyErr_SetString(PyExc_BufferError, + "mmap can't resize with extant buffers exported."); + return 0; + } if ((self->access == ACCESS_WRITE) || (self->access == ACCESS_DEFAULT)) - return 1; + return 1; PyErr_Format(PyExc_TypeError, "mmap can't resize a readonly or copy-on-write memory map."); return 0; @@ -589,53 +600,21 @@ static struct PyMethodDef mmap_object_methods[] = { /* Functions for treating an mmap'ed file as a buffer */ -static Py_ssize_t -mmap_buffer_getreadbuf(mmap_object *self, Py_ssize_t index, const void **ptr) -{ - CHECK_VALID(-1); - if (index != 0) { - PyErr_SetString(PyExc_SystemError, - "Accessing non-existent mmap segment"); - return -1; - } - *ptr = self->data; - return self->size; -} - -static Py_ssize_t -mmap_buffer_getwritebuf(mmap_object *self, Py_ssize_t index, const void **ptr) -{ - CHECK_VALID(-1); - if (index != 0) { - PyErr_SetString(PyExc_SystemError, - "Accessing non-existent mmap segment"); - return -1; - } - if (!is_writeable(self)) - return -1; - *ptr = self->data; - return self->size; -} - -static Py_ssize_t -mmap_buffer_getsegcount(mmap_object *self, Py_ssize_t *lenp) +static int +mmap_buffer_getbuf(mmap_object *self, PyBuffer *view, int flags) { CHECK_VALID(-1); - if (lenp) - *lenp = self->size; - return 1; + if (PyBuffer_FillInfo(view, self->data, self->size, + (self->access == ACCESS_READ), flags) < 0) + return -1; + self->exports++; + return 0; } -static Py_ssize_t -mmap_buffer_getcharbuffer(mmap_object *self, Py_ssize_t index, const void **ptr) +static void +mmap_buffer_releasebuf(mmap_object *self, PyBuffer *view) { - if (index != 0) { - PyErr_SetString(PyExc_SystemError, - "accessing non-existent buffer segment"); - return -1; - } - *ptr = (const char *)self->data; - return self->size; + self->exports--; } static PyObject * @@ -775,10 +754,8 @@ static PySequenceMethods mmap_as_sequence = { }; static PyBufferProcs mmap_as_buffer = { - (readbufferproc)mmap_buffer_getreadbuf, - (writebufferproc)mmap_buffer_getwritebuf, - (segcountproc)mmap_buffer_getsegcount, - (charbufferproc)mmap_buffer_getcharbuffer, + (getbufferproc)mmap_buffer_getbuf, + (releasebufferproc)mmap_buffer_releasebuf, }; static PyTypeObject mmap_object_type = { @@ -900,6 +877,7 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict) m_obj->data = NULL; m_obj->size = (size_t) map_size; m_obj->pos = (size_t) 0; + m_obj->exports = 0; if (fd == -1) { m_obj->fd = -1; /* Assume the caller wants to map anonymous memory. @@ -1069,6 +1047,7 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict) /* set the initial position */ m_obj->pos = (size_t) 0; + m_obj->exports = 0; /* set the tag name */ if (tagname != NULL && *tagname != '\0') { m_obj->tagname = PyMem_Malloc(strlen(tagname)+1); diff --git a/Objects/abstract.c b/Objects/abstract.c index d43bb6a..a48d5dc 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -215,50 +215,50 @@ PyObject_DelItemString(PyObject *o, char *key) return ret; } +/* We release the buffer right after use of this function which could + cause issues later on. Don't use these functions in new code. + */ int PyObject_AsCharBuffer(PyObject *obj, - const char **buffer, - Py_ssize_t *buffer_len) + const char **buffer, + Py_ssize_t *buffer_len) { PyBufferProcs *pb; - char *pp; - Py_ssize_t len; + PyBuffer view; if (obj == NULL || buffer == NULL || buffer_len == NULL) { null_error(); return -1; } pb = obj->ob_type->tp_as_buffer; - if (pb == NULL || - pb->bf_getcharbuffer == NULL || - pb->bf_getsegcount == NULL) { + if (pb == NULL || pb->bf_getbuffer == NULL) { PyErr_SetString(PyExc_TypeError, - "expected a character buffer object"); + "expected an object with the buffer interface"); return -1; - } - if ((*pb->bf_getsegcount)(obj,NULL) != 1) { - PyErr_SetString(PyExc_TypeError, - "expected a single-segment buffer object"); - return -1; - } - len = (*pb->bf_getcharbuffer)(obj, 0, &pp); - if (len < 0) - return -1; - *buffer = pp; - *buffer_len = len; + } + if ((*pb->bf_getbuffer)(obj, &view, PyBUF_CHARACTER)) return -1; + + *buffer = view.buf; + *buffer_len = view.len; + if (pb->bf_releasebuffer != NULL) + (*pb->bf_releasebuffer)(obj, &view); return 0; } int PyObject_CheckReadBuffer(PyObject *obj) { - PyBufferProcs *pb = obj->ob_type->tp_as_buffer; + PyBufferProcs *pb = obj->ob_type->tp_as_buffer; if (pb == NULL || - pb->bf_getreadbuffer == NULL || - pb->bf_getsegcount == NULL || - (*pb->bf_getsegcount)(obj, NULL) != 1) - return 0; + pb->bf_getbuffer == NULL) + return 0; + if ((*pb->bf_getbuffer)(obj, NULL, PyBUF_SIMPLE) == -1) { + PyErr_Clear(); + return 0; + } + if (*pb->bf_releasebuffer != NULL) + (*pb->bf_releasebuffer)(obj, NULL); return 1; } @@ -267,8 +267,7 @@ int PyObject_AsReadBuffer(PyObject *obj, Py_ssize_t *buffer_len) { PyBufferProcs *pb; - void *pp; - Py_ssize_t len; + PyBuffer view; if (obj == NULL || buffer == NULL || buffer_len == NULL) { null_error(); @@ -276,22 +275,18 @@ int PyObject_AsReadBuffer(PyObject *obj, } pb = obj->ob_type->tp_as_buffer; if (pb == NULL || - pb->bf_getreadbuffer == NULL || - pb->bf_getsegcount == NULL) { - PyErr_SetString(PyExc_TypeError, - "expected a readable buffer object"); - return -1; - } - if ((*pb->bf_getsegcount)(obj, NULL) != 1) { + pb->bf_getbuffer == NULL) { PyErr_SetString(PyExc_TypeError, - "expected a single-segment buffer object"); + "expected an object with a buffer interface"); return -1; } - len = (*pb->bf_getreadbuffer)(obj, 0, &pp); - if (len < 0) - return -1; - *buffer = pp; - *buffer_len = len; + + if ((*pb->bf_getbuffer)(obj, &view, PyBUF_SIMPLE)) return -1; + + *buffer = view.buf; + *buffer_len = view.len; + if (pb->bf_releasebuffer != NULL) + (*pb->bf_releasebuffer)(obj, &view); return 0; } @@ -300,8 +295,7 @@ int PyObject_AsWriteBuffer(PyObject *obj, Py_ssize_t *buffer_len) { PyBufferProcs *pb; - void*pp; - Py_ssize_t len; + PyBuffer view; if (obj == NULL || buffer == NULL || buffer_len == NULL) { null_error(); @@ -309,25 +303,384 @@ int PyObject_AsWriteBuffer(PyObject *obj, } pb = obj->ob_type->tp_as_buffer; if (pb == NULL || - pb->bf_getwritebuffer == NULL || - pb->bf_getsegcount == NULL) { - PyErr_SetString(PyExc_TypeError, - "expected a writeable buffer object"); + pb->bf_getbuffer == NULL || + ((*pb->bf_getbuffer)(obj, &view, PyBUF_WRITEABLE) != 0)) { + PyErr_SetString(PyExc_TypeError, + "expected an object with a writeable buffer interface"); return -1; } - if ((*pb->bf_getsegcount)(obj, NULL) != 1) { - PyErr_SetString(PyExc_TypeError, - "expected a single-segment buffer object"); - return -1; - } - len = (*pb->bf_getwritebuffer)(obj,0,&pp); - if (len < 0) - return -1; - *buffer = pp; - *buffer_len = len; + + *buffer = view.buf; + *buffer_len = view.len; + if (pb->bf_releasebuffer != NULL) + (*pb->bf_releasebuffer)(obj, &view); return 0; } +/* Buffer C-API for Python 3.0 */ + +int +PyObject_GetBuffer(PyObject *obj, PyBuffer *view, int flags) +{ + if (!PyObject_CheckBuffer(obj)) { + PyErr_SetString(PyExc_TypeError, + "object does not have the buffer interface"); + return -1; + } + return (*(obj->ob_type->tp_as_buffer->bf_getbuffer))(obj, view, flags); +} + +void +PyObject_ReleaseBuffer(PyObject *obj, PyBuffer *view) +{ + if (obj->ob_type->tp_as_buffer != NULL && + obj->ob_type->tp_as_buffer->bf_releasebuffer != NULL) { + (*(obj->ob_type->tp_as_buffer->bf_releasebuffer))(obj, view); + } +} + + +static int +_IsFortranContiguous(PyBuffer *view) +{ + Py_ssize_t sd, dim; + int i; + + if (view->ndim == 0) return 1; + if (view->strides == NULL) return (view->ndim == 1); + + sd = view->itemsize; + if (view->ndim == 1) return (view->shape[0] == 1 || + sd == view->strides[0]); + for (i=0; indim; i++) { + dim = view->shape[i]; + if (dim == 0) return 1; + if (view->strides[i] != sd) return 0; + sd *= dim; + } + return 1; +} + +static int +_IsCContiguous(PyBuffer *view) +{ + Py_ssize_t sd, dim; + int i; + + if (view->ndim == 0) return 1; + if (view->strides == NULL) return 1; + + sd = view->itemsize; + if (view->ndim == 1) return (view->shape[0] == 1 || + sd == view->strides[0]); + for (i=view->ndim-1; i>=0; i--) { + dim = view->shape[i]; + if (dim == 0) return 1; + if (view->strides[i] != sd) return 0; + sd *= dim; + } + return 1; +} + +int +PyBuffer_IsContiguous(PyBuffer *view, char fort) +{ + + if (view->suboffsets != NULL) return 0; + + if (fort == 'C') + return _IsCContiguous(view); + else if (fort == 'F') + return _IsFortranContiguous(view); + else if (fort == 'A') + return (_IsCContiguous(view) || _IsFortranContiguous(view)); + return 0; +} + + +void* +PyBuffer_GetPointer(PyBuffer *view, Py_ssize_t *indices) +{ + char* pointer; + int i; + pointer = (char *)view->buf; + for (i = 0; i < view->ndim; i++) { + pointer += view->strides[i]*indices[i]; + if ((view->suboffsets != NULL) && (view->suboffsets[i] >= 0)) { + pointer = *((char**)pointer) + view->suboffsets[i]; + } + } + return (void*)pointer; +} + + +void +_add_one_to_index_F(int nd, Py_ssize_t *index, Py_ssize_t *shape) +{ + int k; + + for (k=0; k=0; k--) { + if (index[k] < shape[k]-1) { + index[k]++; + break; + } + else { + index[k] = 0; + } + } +} + + /* view is not checked for consistency in either of these. It is + assumed that the size of the buffer is view->len in + view->len / view->itemsize elements. + */ + +int +PyBuffer_ToContiguous(void *buf, PyBuffer *view, Py_ssize_t len, char fort) +{ + int k; + void (*addone)(int, Py_ssize_t *, Py_ssize_t *); + Py_ssize_t *indices, elements; + char *dest, *ptr; + + if (len > view->len) { + len = view->len; + } + + if (PyBuffer_IsContiguous(view, fort)) { + /* simplest copy is all that is needed */ + memcpy(buf, view->buf, len); + return 0; + } + + /* Otherwise a more elaborate scheme is needed */ + + indices = (Py_ssize_t *)PyMem_Malloc(sizeof(Py_ssize_t)*(view->ndim)); + if (indices == NULL) { + PyErr_NoMemory(); + return -1; + } + for (k=0; kndim;k++) { + indices[k] = 0; + } + + if (fort == 'F') { + addone = _add_one_to_index_F; + } + else { + addone = _add_one_to_index_C; + } + dest = buf; + /* XXX : This is not going to be the fastest code in the world + several optimizations are possible. + */ + elements = len / view->itemsize; + while (elements--) { + addone(view->ndim, indices, view->shape); + ptr = PyBuffer_GetPointer(view, indices); + memcpy(dest, ptr, view->itemsize); + dest += view->itemsize; + } + PyMem_Free(indices); + return 0; +} + +int +PyBuffer_FromContiguous(PyBuffer *view, void *buf, Py_ssize_t len, char fort) +{ + int k; + void (*addone)(int, Py_ssize_t *, Py_ssize_t *); + Py_ssize_t *indices, elements; + char *src, *ptr; + + if (len > view->len) { + len = view->len; + } + + if (PyBuffer_IsContiguous(view, fort)) { + /* simplest copy is all that is needed */ + memcpy(view->buf, buf, len); + return 0; + } + + /* Otherwise a more elaborate scheme is needed */ + + indices = (Py_ssize_t *)PyMem_Malloc(sizeof(Py_ssize_t)*(view->ndim)); + if (indices == NULL) { + PyErr_NoMemory(); + return -1; + } + for (k=0; kndim;k++) { + indices[k] = 0; + } + + if (fort == 'F') { + addone = _add_one_to_index_F; + } + else { + addone = _add_one_to_index_C; + } + src = buf; + /* XXX : This is not going to be the fastest code in the world + several optimizations are possible. + */ + elements = len / view->itemsize; + while (elements--) { + addone(view->ndim, indices, view->shape); + ptr = PyBuffer_GetPointer(view, indices); + memcpy(ptr, src, view->itemsize); + src += view->itemsize; + } + + PyMem_Free(indices); + return 0; +} + +int PyObject_CopyData(PyObject *dest, PyObject *src) +{ + PyBuffer view_dest, view_src; + int k; + Py_ssize_t *indices, elements; + char *dptr, *sptr; + + if (!PyObject_CheckBuffer(dest) || + !PyObject_CheckBuffer(src)) { + PyErr_SetString(PyExc_TypeError, + "both destination and source must have the "\ + "buffer interface"); + return -1; + } + + if (PyObject_GetBuffer(dest, &view_dest, PyBUF_FULL) != 0) return -1; + if (PyObject_GetBuffer(src, &view_src, PyBUF_FULL_RO) != 0) { + PyObject_ReleaseBuffer(dest, &view_dest); + return -1; + } + + if (view_dest.len < view_src.len) { + PyErr_SetString(PyExc_BufferError, + "destination is too small to receive data from source"); + PyObject_ReleaseBuffer(dest, &view_dest); + PyObject_ReleaseBuffer(src, &view_src); + return -1; + } + + if ((PyBuffer_IsContiguous(&view_dest, 'C') && + PyBuffer_IsContiguous(&view_src, 'C')) || + (PyBuffer_IsContiguous(&view_dest, 'F') && + PyBuffer_IsContiguous(&view_src, 'F'))) { + /* simplest copy is all that is needed */ + memcpy(view_dest.buf, view_src.buf, view_src.len); + PyObject_ReleaseBuffer(dest, &view_dest); + PyObject_ReleaseBuffer(src, &view_src); + return 0; + } + + /* Otherwise a more elaborate copy scheme is needed */ + + indices = (Py_ssize_t *)PyMem_Malloc(sizeof(Py_ssize_t)*view_src.ndim); + if (indices == NULL) { + PyErr_NoMemory(); + PyObject_ReleaseBuffer(dest, &view_dest); + PyObject_ReleaseBuffer(src, &view_src); + return -1; + } + for (k=0; k=0; k--) { + strides[k] = sd; + sd *= shape[k]; + } + } + return; +} + +int +PyBuffer_FillInfo(PyBuffer *view, void *buf, Py_ssize_t len, + int readonly, int flags) +{ + if (view == NULL) return 0; + if (((flags & PyBUF_LOCKDATA) == PyBUF_LOCKDATA) && + readonly != -1) { + PyErr_SetString(PyExc_BufferError, + "Cannot make this object read-only."); + return -1; + } + if (((flags & PyBUF_WRITEABLE) == PyBUF_WRITEABLE) && + readonly == 1) { + PyErr_SetString(PyExc_BufferError, + "Object is not writeable."); + return -1; + } + + view->buf = buf; + view->len = len; + view->readonly = readonly; + view->itemsize = 1; + view->format = NULL; + if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) + view->format = "B"; + view->ndim = 1; + view->shape = NULL; + if ((flags & PyBUF_ND) == PyBUF_ND) + view->shape = &(view->len); + view->strides = NULL; + if ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) + view->strides = &(view->itemsize); + view->suboffsets = NULL; + view->internal = NULL; + return 0; +} + /* Operations on numbers */ int diff --git a/Objects/bufferobject.c b/Objects/bufferobject.c index 64b11a9..a8bbd15 100644 --- a/Objects/bufferobject.c +++ b/Objects/bufferobject.c @@ -15,80 +15,58 @@ typedef struct { } PyBufferObject; -enum buffer_t { - READ_BUFFER, - WRITE_BUFFER, - CHAR_BUFFER, - ANY_BUFFER -}; - static int -get_buf(PyBufferObject *self, void **ptr, Py_ssize_t *size, - enum buffer_t buffer_type) +get_buf(PyBufferObject *self, PyBuffer *view, int flags) { if (self->b_base == NULL) { - assert (ptr != NULL); - *ptr = self->b_ptr; - *size = self->b_size; + view->buf = self->b_ptr; + view->len = self->b_size; } else { Py_ssize_t count, offset; - readbufferproc proc = 0; PyBufferProcs *bp = self->b_base->ob_type->tp_as_buffer; - if ((*bp->bf_getsegcount)(self->b_base, NULL) != 1) { - PyErr_SetString(PyExc_TypeError, - "single-segment buffer object expected"); - return 0; - } - if ((buffer_type == READ_BUFFER) || - ((buffer_type == ANY_BUFFER) && self->b_readonly)) - proc = bp->bf_getreadbuffer; - else if ((buffer_type == WRITE_BUFFER) || - (buffer_type == ANY_BUFFER)) - proc = (readbufferproc)bp->bf_getwritebuffer; - else if (buffer_type == CHAR_BUFFER) { - proc = (readbufferproc)bp->bf_getcharbuffer; - } - if (!proc) { - char *buffer_type_name; - switch (buffer_type) { - case READ_BUFFER: - buffer_type_name = "read"; - break; - case WRITE_BUFFER: - buffer_type_name = "write"; - break; - case CHAR_BUFFER: - buffer_type_name = "char"; - break; - default: - buffer_type_name = "no"; - break; - } - PyErr_Format(PyExc_TypeError, - "%s buffer type not available", - buffer_type_name); - return 0; - } - if ((count = (*proc)(self->b_base, 0, ptr)) < 0) - return 0; + if ((*bp->bf_getbuffer)(self->b_base, view, flags) < 0) return 0; + count = view->len; /* apply constraints to the start/end */ if (self->b_offset > count) offset = count; else offset = self->b_offset; - *(char **)ptr = *(char **)ptr + offset; + view->buf = (char*)view->buf + offset; if (self->b_size == Py_END_OF_BUFFER) - *size = count; + view->len = count; else - *size = self->b_size; - if (offset + *size > count) - *size = count - offset; + view->len = self->b_size; + if (offset + view->len > count) + view->len = count - offset; } return 1; } +static int +buffer_getbuf(PyBufferObject *self, PyBuffer *view, int flags) +{ + if (view == NULL) return 0; + if (!get_buf(self, view, flags)) + return -1; + return PyBuffer_FillInfo(view, view->buf, view->len, self->b_readonly, flags); +} + + +static void +buffer_releasebuf(PyBufferObject *self, PyBuffer *view) +{ + /* No-op if there is no self->b_base */ + if (self->b_base != NULL) { + PyBufferProcs *bp = self->b_base->ob_type->tp_as_buffer; + if (bp->bf_releasebuffer != NULL) { + (*bp->bf_releasebuffer)(self->b_base, view); + } + } + return; +} + static PyObject * buffer_from_memory(PyObject *base, Py_ssize_t size, Py_ssize_t offset, void *ptr, int readonly) @@ -152,10 +130,8 @@ PyBuffer_FromObject(PyObject *base, Py_ssize_t offset, Py_ssize_t size) PyBufferProcs *pb = base->ob_type->tp_as_buffer; if ( pb == NULL || - pb->bf_getreadbuffer == NULL || - pb->bf_getsegcount == NULL ) - { - PyErr_SetString(PyExc_TypeError, "buffer object expected"); + pb->bf_getbuffer == NULL) { + PyErr_SetString(PyExc_TypeError, "buffer object expected"); return NULL; } @@ -168,9 +144,7 @@ PyBuffer_FromReadWriteObject(PyObject *base, Py_ssize_t offset, Py_ssize_t size) PyBufferProcs *pb = base->ob_type->tp_as_buffer; if ( pb == NULL || - pb->bf_getwritebuffer == NULL || - pb->bf_getsegcount == NULL ) - { + pb->bf_getbuffer == NULL) { PyErr_SetString(PyExc_TypeError, "buffer object expected"); return NULL; } @@ -252,12 +226,12 @@ buffer_dealloc(PyBufferObject *self) } static int -get_bufx(PyObject *obj, void **ptr, Py_ssize_t *size) +get_bufx(PyObject *obj, PyBuffer *view, int flags) { PyBufferProcs *bp; if (PyBuffer_Check(obj)) { - if (!get_buf((PyBufferObject *)obj, ptr, size, ANY_BUFFER)) { + if (!get_buf((PyBufferObject *)obj, view, flags)) { PyErr_Clear(); return 0; } @@ -266,17 +240,11 @@ get_bufx(PyObject *obj, void **ptr, Py_ssize_t *size) } bp = obj->ob_type->tp_as_buffer; if (bp == NULL || - bp->bf_getreadbuffer == NULL || - bp->bf_getsegcount == NULL) - return 0; - if ((*bp->bf_getsegcount)(obj, NULL) != 1) + bp->bf_getbuffer == NULL) return 0; - *size = (*bp->bf_getreadbuffer)(obj, 0, ptr); - if (*size < 0) { - PyErr_Clear(); + if ((*bp->bf_getbuffer)(obj, view, PyBUF_SIMPLE) < 0) return 0; - } - return 1; + return 1; } static PyObject * @@ -285,12 +253,15 @@ buffer_richcompare(PyObject *self, PyObject *other, int op) void *p1, *p2; Py_ssize_t len1, len2, min_len; int cmp, ok; + PyBuffer v1, v2; ok = 1; - if (!get_bufx(self, &p1, &len1)) + if (!get_bufx(self, &v1, PyBUF_SIMPLE)) ok = 0; - if (!get_bufx(other, &p2, &len2)) + if (!get_bufx(other, &v2, PyBUF_SIMPLE)) { + if (ok) PyObject_ReleaseBuffer((PyObject *)self, &v1); ok = 0; + } if (!ok) { /* If we can't get the buffers, == and != are still defined @@ -305,11 +276,17 @@ buffer_richcompare(PyObject *self, PyObject *other, int op) Py_INCREF(result); return result; } + len1 = v1.len; + len2 = v2.len; + p1 = v1.buf; + p2 = v2.buf; min_len = (len1 < len2) ? len1 : len2; cmp = memcmp(p1, p2, min_len); if (cmp == 0) cmp = (len1 < len2) ? -1 : (len1 > len2) ? 1 : 0; + PyObject_ReleaseBuffer((PyObject *)self, &v1); + PyObject_ReleaseBuffer(other, &v2); return Py_CmpToRich(op, cmp); } @@ -337,8 +314,7 @@ buffer_repr(PyBufferObject *self) static long buffer_hash(PyBufferObject *self) { - void *ptr; - Py_ssize_t size; + PyBuffer view; register Py_ssize_t len; register unsigned char *p; register long x; @@ -346,42 +322,39 @@ buffer_hash(PyBufferObject *self) if ( self->b_hash != -1 ) return self->b_hash; - /* XXX potential bugs here, a readonly buffer does not imply that the - * underlying memory is immutable. b_readonly is a necessary but not - * sufficient condition for a buffer to be hashable. Perhaps it would - * be better to only allow hashing if the underlying object is known to - * be immutable (e.g. PyString_Check() is true). Another idea would - * be to call tp_hash on the underlying object and see if it raises - * an error. */ - if ( !self->b_readonly ) - { - PyErr_SetString(PyExc_TypeError, - "writable buffers are not hashable"); + if (!get_buf(self, &view, PyBUF_SIMPLE)) return -1; - } - - if (!get_buf(self, &ptr, &size, ANY_BUFFER)) + if (!(self->b_readonly)) { + PyErr_SetString(PyExc_TypeError, + "writable buffers are not hashable"); + PyObject_ReleaseBuffer((PyObject *)self, &view); return -1; - p = (unsigned char *) ptr; - len = size; + } + + p = (unsigned char *) view.buf; + len = view.len; x = *p << 7; while (--len >= 0) x = (1000003*x) ^ *p++; - x ^= size; + x ^= view.len; if (x == -1) x = -2; self->b_hash = x; + PyObject_ReleaseBuffer((PyObject *)self, &view); return x; } static PyObject * buffer_str(PyBufferObject *self) { - void *ptr; - Py_ssize_t size; - if (!get_buf(self, &ptr, &size, ANY_BUFFER)) + PyBuffer view; + PyObject *res; + + if (!get_buf(self, &view, PyBUF_SIMPLE)) return NULL; - return PyString_FromStringAndSize((const char *)ptr, size); + res = PyString_FromStringAndSize((const char *)view.buf, view.len); + PyObject_ReleaseBuffer((PyObject *)self, &view); + return res; } /* Sequence methods */ @@ -389,71 +362,58 @@ buffer_str(PyBufferObject *self) static Py_ssize_t buffer_length(PyBufferObject *self) { - void *ptr; - Py_ssize_t size; - if (!get_buf(self, &ptr, &size, ANY_BUFFER)) + PyBuffer view; + + if (!get_buf(self, &view, PyBUF_SIMPLE)) return -1; - return size; + PyObject_ReleaseBuffer((PyObject *)self, &view); + return view.len; } static PyObject * buffer_concat(PyBufferObject *self, PyObject *other) { PyBufferProcs *pb = other->ob_type->tp_as_buffer; - void *ptr1, *ptr2; char *p; PyObject *ob; - Py_ssize_t size, count; + PyBuffer view, view2; if ( pb == NULL || - pb->bf_getreadbuffer == NULL || - pb->bf_getsegcount == NULL ) + pb->bf_getbuffer == NULL) { PyErr_BadArgument(); return NULL; } - if ( (*pb->bf_getsegcount)(other, NULL) != 1 ) - { - /* ### use a different exception type/message? */ - PyErr_SetString(PyExc_TypeError, - "single-segment buffer object expected"); - return NULL; - } - if (!get_buf(self, &ptr1, &size, ANY_BUFFER)) + if (!get_buf(self, &view, PyBUF_SIMPLE)) return NULL; /* optimize special case */ /* XXX bad idea type-wise */ - if ( size == 0 ) - { - Py_INCREF(other); - return other; + if ( view.len == 0 ) { + PyObject_ReleaseBuffer((PyObject *)self, &view); + Py_INCREF(other); + return other; } - if (PyUnicode_Check(other)) { - /* XXX HACK */ - if ( (count = (*pb->bf_getcharbuffer)(other, 0, - (char **)&ptr2)) < 0 ) - return NULL; - } - else { - if ( (count = (*pb->bf_getreadbuffer)(other, 0, &ptr2)) < 0 ) - return NULL; - } + if (PyObject_GetBuffer((PyObject *)other, &view2, PyBUF_SIMPLE) < 0) { + PyObject_ReleaseBuffer((PyObject *)self, &view); + return NULL; + } - /* XXX Should return a bytes object, really */ - ob = PyString_FromStringAndSize(NULL, size + count); - if ( ob == NULL ) + ob = PyBytes_FromStringAndSize(NULL, view.len+view2.len); + if ( ob == NULL ) { + PyObject_ReleaseBuffer((PyObject *)self, &view); + PyObject_ReleaseBuffer(other, &view2); return NULL; - p = PyString_AS_STRING(ob); - memcpy(p, ptr1, size); - memcpy(p + size, ptr2, count); - - /* there is an extra byte in the string object, so this is safe */ - p[size + count] = '\0'; - - return ob; + } + p = PyBytes_AS_STRING(ob); + memcpy(p, view.buf, view.len); + memcpy(p + view.len, view2.buf, view2.len); + + PyObject_ReleaseBuffer((PyObject *)self, &view); + PyObject_ReleaseBuffer(other, &view2); + return ob; } static PyObject * @@ -461,81 +421,83 @@ buffer_repeat(PyBufferObject *self, Py_ssize_t count) { PyObject *ob; register char *p; - void *ptr; - Py_ssize_t size; + PyBuffer view; if ( count < 0 ) count = 0; - if (!get_buf(self, &ptr, &size, ANY_BUFFER)) + if (!get_buf(self, &view, PyBUF_SIMPLE)) return NULL; - ob = PyString_FromStringAndSize(NULL, size * count); + ob = PyBytes_FromStringAndSize(NULL, view.len * count); if ( ob == NULL ) return NULL; - p = PyString_AS_STRING(ob); + p = PyBytes_AS_STRING(ob); while ( count-- ) { - memcpy(p, ptr, size); - p += size; + memcpy(p, view.buf, view.len); + p += view.len; } - /* there is an extra byte in the string object, so this is safe */ - *p = '\0'; - + PyObject_ReleaseBuffer((PyObject *)self, &view); return ob; } static PyObject * buffer_item(PyBufferObject *self, Py_ssize_t idx) { - void *ptr; - Py_ssize_t size; - if (!get_buf(self, &ptr, &size, ANY_BUFFER)) + PyBuffer view; + PyObject *ob; + + if (!get_buf(self, &view, PyBUF_SIMPLE)) return NULL; - if ( idx < 0 || idx >= size ) { + if ( idx < 0 || idx >= view.len ) { PyErr_SetString(PyExc_IndexError, "buffer index out of range"); return NULL; } - return PyString_FromStringAndSize((char *)ptr + idx, 1); + ob = PyBytes_FromStringAndSize((char *)view.buf + idx, 1); + PyObject_ReleaseBuffer((PyObject *)self, &view); + return ob; } static PyObject * buffer_slice(PyBufferObject *self, Py_ssize_t left, Py_ssize_t right) { - void *ptr; - Py_ssize_t size; - if (!get_buf(self, &ptr, &size, ANY_BUFFER)) + PyObject *ob; + PyBuffer view; + if (!get_buf(self, &view, PyBUF_SIMPLE)) return NULL; if ( left < 0 ) left = 0; if ( right < 0 ) right = 0; - if ( right > size ) - right = size; + if ( right > view.len ) + right = view.len; if ( right < left ) right = left; - return PyString_FromStringAndSize((char *)ptr + left, - right - left); + ob = PyBytes_FromStringAndSize((char *)view.buf + left, + right - left); + PyObject_ReleaseBuffer((PyObject *)self, &view); + return ob; } static int buffer_ass_item(PyBufferObject *self, Py_ssize_t idx, PyObject *other) { PyBufferProcs *pb; - void *ptr1, *ptr2; - Py_ssize_t size; - Py_ssize_t count; + PyBuffer view, view2; - if ( self->b_readonly ) { + if (!get_buf(self, &view, PyBUF_SIMPLE)) + return -1; + + if ( self->b_readonly || view.readonly ) { PyErr_SetString(PyExc_TypeError, "buffer is read-only"); + PyObject_ReleaseBuffer((PyObject *)self, &view); return -1; } - if (!get_buf(self, &ptr1, &size, ANY_BUFFER)) - return -1; - - if (idx < 0 || idx >= size) { + if (idx < 0 || idx >= view.len) { + PyObject_ReleaseBuffer((PyObject *)self, &view); PyErr_SetString(PyExc_IndexError, "buffer assignment index out of range"); return -1; @@ -543,29 +505,27 @@ buffer_ass_item(PyBufferObject *self, Py_ssize_t idx, PyObject *other) pb = other ? other->ob_type->tp_as_buffer : NULL; if ( pb == NULL || - pb->bf_getreadbuffer == NULL || - pb->bf_getsegcount == NULL ) - { + pb->bf_getbuffer == NULL) { PyErr_BadArgument(); + PyObject_ReleaseBuffer((PyObject *)self, &view); return -1; } - if ( (*pb->bf_getsegcount)(other, NULL) != 1 ) - { - /* ### use a different exception type/message? */ - PyErr_SetString(PyExc_TypeError, - "single-segment buffer object expected"); - return -1; - } - - if ( (count = (*pb->bf_getreadbuffer)(other, 0, &ptr2)) < 0 ) - return -1; - if ( count != 1 ) { + + if (PyObject_GetBuffer(other, &view2, PyBUF_SIMPLE) < 0) { + PyObject_ReleaseBuffer((PyObject *)self, &view); + return -1; + } + if ( view.len != 1 ) { + PyObject_ReleaseBuffer((PyObject *)self, &view); + PyObject_ReleaseBuffer(other, &view2); PyErr_SetString(PyExc_TypeError, "right operand must be a single byte"); return -1; } - ((char *)ptr1)[idx] = *(char *)ptr2; + ((char *)(view.buf))[idx] = *((char *)(view2.buf)); + PyObject_ReleaseBuffer((PyObject *)self, &view); + PyObject_ReleaseBuffer(other, &view2); return 0; } @@ -573,125 +533,60 @@ static int buffer_ass_slice(PyBufferObject *self, Py_ssize_t left, Py_ssize_t right, PyObject *other) { PyBufferProcs *pb; - void *ptr1, *ptr2; - Py_ssize_t size; + PyBuffer v1, v2; Py_ssize_t slice_len; - Py_ssize_t count; - - if ( self->b_readonly ) { - PyErr_SetString(PyExc_TypeError, - "buffer is read-only"); - return -1; - } pb = other ? other->ob_type->tp_as_buffer : NULL; if ( pb == NULL || - pb->bf_getreadbuffer == NULL || - pb->bf_getsegcount == NULL ) + pb->bf_getbuffer == NULL) { PyErr_BadArgument(); return -1; } - if ( (*pb->bf_getsegcount)(other, NULL) != 1 ) - { - /* ### use a different exception type/message? */ + if (!get_buf(self, &v1, PyBUF_SIMPLE)) + return -1; + + if ( self->b_readonly || v1.readonly) { PyErr_SetString(PyExc_TypeError, - "single-segment buffer object expected"); + "buffer is read-only"); + PyObject_ReleaseBuffer((PyObject *)self, &v1); return -1; } - if (!get_buf(self, &ptr1, &size, ANY_BUFFER)) - return -1; - if ( (count = (*pb->bf_getreadbuffer)(other, 0, &ptr2)) < 0 ) - return -1; + + if ((*pb->bf_getbuffer)(other, &v2, PyBUF_SIMPLE) < 0) { + PyObject_ReleaseBuffer((PyObject *)self, &v1); + return -1; + } if ( left < 0 ) left = 0; - else if ( left > size ) - left = size; + else if ( left > v1.len ) + left = v1.len; if ( right < left ) right = left; - else if ( right > size ) - right = size; + else if ( right > v1.len ) + right = v1.len; slice_len = right - left; - if ( count != slice_len ) { + if ( v2.len != slice_len ) { PyErr_SetString( PyExc_TypeError, "right operand length must match slice length"); + PyObject_ReleaseBuffer((PyObject *)self, &v1); + PyObject_ReleaseBuffer(other, &v2); return -1; } if ( slice_len ) - memcpy((char *)ptr1 + left, ptr2, slice_len); + memcpy((char *)v1.buf + left, v2.buf, slice_len); + PyObject_ReleaseBuffer((PyObject *)self, &v1); + PyObject_ReleaseBuffer(other, &v2); return 0; } /* Buffer methods */ -static Py_ssize_t -buffer_getreadbuf(PyBufferObject *self, Py_ssize_t idx, void **pp) -{ - Py_ssize_t size; - if ( idx != 0 ) { - PyErr_SetString(PyExc_SystemError, - "accessing non-existent buffer segment"); - return -1; - } - if (!get_buf(self, pp, &size, READ_BUFFER)) - return -1; - return size; -} - -static Py_ssize_t -buffer_getwritebuf(PyBufferObject *self, Py_ssize_t idx, void **pp) -{ - Py_ssize_t size; - - if ( self->b_readonly ) - { - PyErr_SetString(PyExc_TypeError, "buffer is read-only"); - return -1; - } - - if ( idx != 0 ) { - PyErr_SetString(PyExc_SystemError, - "accessing non-existent buffer segment"); - return -1; - } - if (!get_buf(self, pp, &size, WRITE_BUFFER)) - return -1; - return size; -} - -static Py_ssize_t -buffer_getsegcount(PyBufferObject *self, Py_ssize_t *lenp) -{ - void *ptr; - Py_ssize_t size; - if (!get_buf(self, &ptr, &size, ANY_BUFFER)) - return -1; - if (lenp) - *lenp = size; - return 1; -} - -static Py_ssize_t -buffer_getcharbuf(PyBufferObject *self, Py_ssize_t idx, const char **pp) -{ - void *ptr; - Py_ssize_t size; - if ( idx != 0 ) { - PyErr_SetString(PyExc_SystemError, - "accessing non-existent buffer segment"); - return -1; - } - if (!get_buf(self, &ptr, &size, CHAR_BUFFER)) - return -1; - *pp = (const char *)ptr; - return size; -} - static PySequenceMethods buffer_as_sequence = { (lenfunc)buffer_length, /*sq_length*/ (binaryfunc)buffer_concat, /*sq_concat*/ @@ -703,10 +598,8 @@ static PySequenceMethods buffer_as_sequence = { }; static PyBufferProcs buffer_as_buffer = { - (readbufferproc)buffer_getreadbuf, - (writebufferproc)buffer_getwritebuf, - (segcountproc)buffer_getsegcount, - (charbufferproc)buffer_getcharbuf, + (getbufferproc)buffer_getbuf, + (releasebufferproc)buffer_releasebuf, }; PyTypeObject PyBuffer_Type = { diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index b409a28..5a03beb 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -26,6 +26,7 @@ PyBytes_Init(void) return 0; nullbytes->ob_bytes = NULL; Py_Size(nullbytes) = nullbytes->ob_alloc = 0; + nullbytes->ob_exports = 0; return 1; } @@ -48,22 +49,44 @@ _getbytevalue(PyObject* arg, int *value) return 1; } +static int +bytes_getbuffer(PyBytesObject *obj, PyBuffer *view, int flags) +{ + int ret; + void *ptr; + if (view == NULL) { + obj->ob_exports++; + return 0; + } + if (obj->ob_bytes == NULL) + ptr = ""; + else + ptr = obj->ob_bytes; + ret = PyBuffer_FillInfo(view, ptr, Py_Size(obj), 0, flags); + if (ret >= 0) { + obj->ob_exports++; + } + return ret; +} + +static void +bytes_releasebuffer(PyBytesObject *obj, PyBuffer *view) +{ + obj->ob_exports--; +} + Py_ssize_t -_getbuffer(PyObject *obj, void **ptr) +_getbuffer(PyObject *obj, PyBuffer *view) { PyBufferProcs *buffer = Py_Type(obj)->tp_as_buffer; if (buffer == NULL || PyUnicode_Check(obj) || - buffer->bf_getreadbuffer == NULL || - buffer->bf_getsegcount == NULL || - buffer->bf_getsegcount(obj, NULL) != 1) - { - *ptr = NULL; - return -1; - } + buffer->bf_getbuffer == NULL) return -1; - return buffer->bf_getreadbuffer(obj, 0, ptr); + if (buffer->bf_getbuffer(obj, view, PyBUF_SIMPLE) < 0) + return -1; + return view->len; } /* Direct API functions */ @@ -104,6 +127,7 @@ PyBytes_FromStringAndSize(const char *bytes, Py_ssize_t size) } Py_Size(new) = size; new->ob_alloc = alloc; + new->ob_exports = 0; return (PyObject *)new; } @@ -155,6 +179,15 @@ PyBytes_Resize(PyObject *self, Py_ssize_t size) alloc = size + 1; } + if (((PyBytesObject *)self)->ob_exports > 0) { + /* + fprintf(stderr, "%d: %s", ((PyBytesObject *)self)->ob_exports, ((PyBytesObject *)self)->ob_bytes); + */ + PyErr_SetString(PyExc_BufferError, + "Existing exports of data: object cannot be re-sized"); + return -1; + } + sval = PyMem_Realloc(((PyBytesObject *)self)->ob_bytes, alloc); if (sval == NULL) { PyErr_NoMemory(); @@ -172,27 +205,38 @@ PyBytes_Resize(PyObject *self, Py_ssize_t size) PyObject * PyBytes_Concat(PyObject *a, PyObject *b) { - Py_ssize_t asize, bsize, size; - void *aptr, *bptr; + Py_ssize_t size; + PyBuffer va, vb; PyBytesObject *result; - asize = _getbuffer(a, &aptr); - bsize = _getbuffer(b, &bptr); - if (asize < 0 || bsize < 0) { - PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", - Py_Type(a)->tp_name, Py_Type(b)->tp_name); - return NULL; + va.len = -1; + vb.len = -1; + if (_getbuffer(a, &va) < 0 || + _getbuffer(b, &vb) < 0) { + if (va.len != -1) + PyObject_ReleaseBuffer(a, &va); + if (vb.len != -1) + PyObject_ReleaseBuffer(b, &vb); + PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", + Py_Type(a)->tp_name, Py_Type(b)->tp_name); + return NULL; } - size = asize + bsize; - if (size < 0) - return PyErr_NoMemory(); + size = va.len + vb.len; + if (size < 0) { + PyObject_ReleaseBuffer(a, &va); + PyObject_ReleaseBuffer(b, &vb); + return PyErr_NoMemory(); + } result = (PyBytesObject *) PyBytes_FromStringAndSize(NULL, size); if (result != NULL) { - memcpy(result->ob_bytes, aptr, asize); - memcpy(result->ob_bytes + asize, bptr, bsize); + memcpy(result->ob_bytes, va.buf, va.len); + memcpy(result->ob_bytes + va.len, vb.buf, vb.len); } + + PyObject_ReleaseBuffer(a, &va); + PyObject_ReleaseBuffer(b, &vb); return (PyObject *)result; } @@ -213,30 +257,32 @@ bytes_concat(PyBytesObject *self, PyObject *other) static PyObject * bytes_iconcat(PyBytesObject *self, PyObject *other) { - void *optr; - Py_ssize_t osize; Py_ssize_t mysize; Py_ssize_t size; + PyBuffer vo; - /* XXX What if other == self? */ - osize = _getbuffer(other, &optr); - if (osize < 0) { - PyErr_Format(PyExc_TypeError, - "can't concat bytes to %.100s", Py_Type(other)->tp_name); - return NULL; + if (_getbuffer(other, &vo) < 0) { + PyErr_Format(PyExc_TypeError, + "can't concat bytes to %.100s", Py_Type(self)->tp_name); + return NULL; } mysize = Py_Size(self); - size = mysize + osize; - if (size < 0) - return PyErr_NoMemory(); + size = mysize + vo.len; + if (size < 0) { + PyObject_ReleaseBuffer(other, &vo); + return PyErr_NoMemory(); + } if (size < self->ob_alloc) { - Py_Size(self) = size; - self->ob_bytes[Py_Size(self)] = '\0'; /* Trailing null byte */ + Py_Size(self) = size; + self->ob_bytes[Py_Size(self)] = '\0'; /* Trailing null byte */ } - else if (PyBytes_Resize((PyObject *)self, size) < 0) - return NULL; - memcpy(self->ob_bytes + mysize, optr, osize); + else if (PyBytes_Resize((PyObject *)self, size) < 0) { + PyObject_ReleaseBuffer(other, &vo); + return NULL; + } + memcpy(self->ob_bytes + mysize, vo.buf, vo.len); + PyObject_ReleaseBuffer(other, &vo); Py_INCREF(self); return (PyObject *)self; } @@ -409,9 +455,12 @@ bytes_setslice(PyBytesObject *self, Py_ssize_t lo, Py_ssize_t hi, { Py_ssize_t avail, needed; void *bytes; + PyBuffer vbytes; + int res = 0; + vbytes.len = -1; if (values == (PyObject *)self) { - /* Make a copy an call this function recursively */ + /* Make a copy and call this function recursively */ int err; values = PyBytes_FromObject(values); if (values == NULL) @@ -426,13 +475,14 @@ bytes_setslice(PyBytesObject *self, Py_ssize_t lo, Py_ssize_t hi, needed = 0; } else { - needed = _getbuffer(values, &bytes); - if (needed < 0) { - PyErr_Format(PyExc_TypeError, - "can't set bytes slice from %.100s", - Py_Type(values)->tp_name); - return -1; - } + if (_getbuffer(values, &vbytes) < 0) { + PyErr_Format(PyExc_TypeError, + "can't set bytes slice from %.100s", + Py_Type(values)->tp_name); + return -1; + } + needed = vbytes.len; + bytes = vbytes.buf; } if (lo < 0) @@ -458,8 +508,10 @@ bytes_setslice(PyBytesObject *self, Py_ssize_t lo, Py_ssize_t hi, Py_Size(self) - hi); } if (PyBytes_Resize((PyObject *)self, - Py_Size(self) + needed - avail) < 0) - return -1; + Py_Size(self) + needed - avail) < 0) { + res = -1; + goto finish; + } if (avail < needed) { /* 0 lo hi old_size @@ -475,7 +527,11 @@ bytes_setslice(PyBytesObject *self, Py_ssize_t lo, Py_ssize_t hi, if (needed > 0) memcpy(self->ob_bytes + lo, bytes, needed); - return 0; + + finish: + if (vbytes.len != -1) + PyObject_ReleaseBuffer(values, &vbytes); + return res; } static int @@ -743,16 +799,22 @@ bytes_init(PyBytesObject *self, PyObject *args, PyObject *kwds) } return 0; } - - if (PyObject_CheckReadBuffer(arg)) { - const void *bytes; + + /* Use the modern buffer interface */ + if (PyObject_CheckBuffer(arg)) { Py_ssize_t size; - if (PyObject_AsReadBuffer(arg, &bytes, &size) < 0) + PyBuffer view; + if (PyObject_GetBuffer(arg, &view, PyBUF_FULL_RO) < 0) return -1; - if (PyBytes_Resize((PyObject *)self, size) < 0) - return -1; - memcpy(self->ob_bytes, bytes, size); + size = view.len; + if (PyBytes_Resize((PyObject *)self, size) < 0) goto fail; + if (PyBuffer_ToContiguous(self->ob_bytes, &view, size, 'C') < 0) + goto fail; + PyObject_ReleaseBuffer(arg, &view); return 0; + fail: + PyObject_ReleaseBuffer(arg, &view); + return -1; } /* XXX Optimize this if the arguments is a list, tuple */ @@ -881,7 +943,7 @@ static PyObject * bytes_richcompare(PyObject *self, PyObject *other, int op) { Py_ssize_t self_size, other_size; - void *self_bytes, *other_bytes; + PyBuffer self_bytes, other_bytes; PyObject *res; Py_ssize_t minsize; int cmp; @@ -897,6 +959,7 @@ bytes_richcompare(PyObject *self, PyObject *other, int op) other_size = _getbuffer(other, &other_bytes); if (other_size < 0) { + PyObject_ReleaseBuffer(self, &self_bytes); Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } @@ -910,7 +973,7 @@ bytes_richcompare(PyObject *self, PyObject *other, int op) if (other_size < minsize) minsize = other_size; - cmp = memcmp(self_bytes, other_bytes, minsize); + cmp = memcmp(self_bytes.buf, other_bytes.buf, minsize); /* In ISO C, memcmp() guarantees to use unsigned bytes! */ if (cmp == 0) { @@ -931,6 +994,8 @@ bytes_richcompare(PyObject *self, PyObject *other, int op) } res = cmp ? Py_True : Py_False; + PyObject_ReleaseBuffer(self, &self_bytes); + PyObject_ReleaseBuffer(other, &other_bytes); Py_INCREF(res); return res; } @@ -944,30 +1009,6 @@ bytes_dealloc(PyBytesObject *self) Py_Type(self)->tp_free((PyObject *)self); } -static Py_ssize_t -bytes_getbuffer(PyBytesObject *self, Py_ssize_t index, const void **ptr) -{ - if (index != 0) { - PyErr_SetString(PyExc_SystemError, - "accessing non-existent bytes segment"); - return -1; - } - if (self->ob_bytes == NULL) - *ptr = ""; - else - *ptr = self->ob_bytes; - return Py_Size(self); -} - -static Py_ssize_t -bytes_getsegcount(PyStringObject *self, Py_ssize_t *lenp) -{ - if (lenp) - *lenp = Py_Size(self); - return 1; -} - - /* -------------------------------------------------------------------- */ /* Methods */ @@ -1018,6 +1059,7 @@ bytes_find_internal(PyBytesObject *self, PyObject *args, int dir) sub = PyBytes_AS_STRING(subobj); sub_len = PyBytes_GET_SIZE(subobj); } + /* XXX --> use the modern buffer interface */ else if (PyObject_AsCharBuffer(subobj, &sub, &sub_len)) /* XXX - the "expected a character buffer object" is pretty confusing for a non-expert. remap to something else ? */ @@ -1075,6 +1117,7 @@ bytes_count(PyBytesObject *self, PyObject *args) sub = PyBytes_AS_STRING(sub_obj); sub_len = PyBytes_GET_SIZE(sub_obj); } + /* XXX --> use the modern buffer interface */ else if (PyObject_AsCharBuffer(sub_obj, &sub, &sub_len)) return NULL; @@ -1162,6 +1205,7 @@ _bytes_tailmatch(PyBytesObject *self, PyObject *substr, Py_ssize_t start, sub = PyBytes_AS_STRING(substr); slen = PyBytes_GET_SIZE(substr); } + /* XXX --> Use the modern buffer interface */ else if (PyObject_AsCharBuffer(substr, &sub, &slen)) return -1; str = PyBytes_AS_STRING(self); @@ -1297,6 +1341,7 @@ bytes_translate(PyBytesObject *self, PyObject *args) table1 = PyBytes_AS_STRING(tableobj); tablen = PyBytes_GET_SIZE(tableobj); } + /* XXX -> Use the modern buffer interface */ else if (PyObject_AsCharBuffer(tableobj, &table1, &tablen)) return NULL; @@ -1311,6 +1356,7 @@ bytes_translate(PyBytesObject *self, PyObject *args) del_table = PyBytes_AS_STRING(delobj); dellen = PyBytes_GET_SIZE(delobj); } + /* XXX -> use the modern buffer interface */ else if (PyObject_AsCharBuffer(delobj, &del_table, &dellen)) return NULL; } @@ -1973,9 +2019,11 @@ static PyObject * bytes_replace(PyBytesObject *self, PyObject *args) { Py_ssize_t count = -1; - PyObject *from, *to; + PyObject *from, *to, *res; const char *from_s, *to_s; Py_ssize_t from_len, to_len; + int relfrom=0, relto=0; + PyBuffer vfrom, vto; if (!PyArg_ParseTuple(args, "OO|n:replace", &from, &to, &count)) return NULL; @@ -1984,19 +2032,38 @@ bytes_replace(PyBytesObject *self, PyObject *args) from_s = PyBytes_AS_STRING(from); from_len = PyBytes_GET_SIZE(from); } - else if (PyObject_AsCharBuffer(from, &from_s, &from_len)) - return NULL; + else { + if (PyObject_GetBuffer(from, &vfrom, PyBUF_CHARACTER) < 0) + return NULL; + from_s = vfrom.buf; + from_len = vfrom.len; + relfrom = 1; + } if (PyBytes_Check(to)) { to_s = PyBytes_AS_STRING(to); to_len = PyBytes_GET_SIZE(to); } - else if (PyObject_AsCharBuffer(to, &to_s, &to_len)) - return NULL; + else { + if (PyObject_GetBuffer(to, &vto, PyBUF_CHARACTER) < 0) { + if (relfrom) + PyObject_ReleaseBuffer(from, &vfrom); + return NULL; + } + to_s = vto.buf; + to_len = vto.len; + relto = 1; + } - return (PyObject *)replace((PyBytesObject *) self, - from_s, from_len, - to_s, to_len, count); + res = (PyObject *)replace((PyBytesObject *) self, + from_s, from_len, + to_s, to_len, count); + + if (relfrom) + PyObject_ReleaseBuffer(from, &vfrom); + if (relto) + PyObject_ReleaseBuffer(to, &vto); + return res; } @@ -2104,6 +2171,7 @@ bytes_split(PyBytesObject *self, PyObject *args) sub = PyBytes_AS_STRING(subobj); n = PyBytes_GET_SIZE(subobj); } + /* XXX -> use the modern buffer interface */ else if (PyObject_AsCharBuffer(subobj, &sub, &n)) return NULL; @@ -2261,6 +2329,7 @@ bytes_rsplit(PyBytesObject *self, PyObject *args) sub = PyBytes_AS_STRING(subobj); n = PyBytes_GET_SIZE(subobj); } + /* XXX -> Use the modern buffer interface */ else if (PyObject_AsCharBuffer(subobj, &sub, &n)) return NULL; @@ -2756,12 +2825,8 @@ static PyMappingMethods bytes_as_mapping = { }; static PyBufferProcs bytes_as_buffer = { - (readbufferproc)bytes_getbuffer, - (writebufferproc)bytes_getbuffer, - (segcountproc)bytes_getsegcount, - /* XXX Bytes are not characters! But we need to implement - bf_getcharbuffer() so we can be used as 't#' argument to codecs. */ - (charbufferproc)bytes_getbuffer, + (getbufferproc)bytes_getbuffer, + (releasebufferproc)bytes_releasebuffer, }; static PyMethodDef diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 2d096a7..710495f 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -1539,6 +1539,11 @@ SimpleExtendsException(PyExc_Exception, ReferenceError, */ SimpleExtendsException(PyExc_Exception, MemoryError, "Out of memory."); +/* + * BufferError extends Exception + */ +SimpleExtendsException(PyExc_Exception, BufferError, "Buffer error."); + /* Warning category docstrings */ diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c new file mode 100644 index 0000000..efcb7ae --- /dev/null +++ b/Objects/memoryobject.c @@ -0,0 +1,540 @@ + +/* Memoryview object implementation */ + +#include "Python.h" + +static int +memory_getbuf(PyMemoryViewObject *self, PyBuffer *view, int flags) +{ + if (view != NULL) + memcpy(view, &(self->view), sizeof(PyBuffer)); + return self->base->ob_type->tp_as_buffer->bf_getbuffer(self->base, + NULL, PyBUF_FULL); +} + +static void +memory_releasebuf(PyMemoryViewObject *self, PyBuffer *view) +{ + PyObject_ReleaseBuffer(self->base, NULL); +} + +PyDoc_STRVAR(memory_doc, +"memoryview(object)\n\ +\n\ +Create a new memoryview object which references the given object."); + +PyObject * +PyMemoryView_FromMemory(PyBuffer *info) +{ + return NULL; +} + +PyObject * +PyMemoryView_FromObject(PyObject *base) +{ + PyMemoryViewObject *mview; + + if (!PyObject_CheckBuffer(base)) { + PyErr_SetString(PyExc_TypeError, + "cannot make memory view because object does "\ + "not have the buffer interface"); + return NULL; + } + + mview = (PyMemoryViewObject *)PyObject_New(PyMemoryViewObject, + &PyMemoryView_Type); + if (mview == NULL) return NULL; + + if (PyObject_GetBuffer(base, &(mview->view), PyBUF_FULL) < 0) { + PyObject_DEL(mview); + return NULL; + } + + mview->base = base; + Py_INCREF(base); + return (PyObject *)mview; +} + +static PyObject * +memory_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) +{ + PyObject *obj; + if (!PyArg_ParseTuple(args, "O", &obj)) return NULL; + + return PyMemoryView_FromObject(obj); +} + + +static void +_strided_copy_nd(char *dest, char *src, int nd, Py_ssize_t *shape, + Py_ssize_t *strides, int itemsize, char fort) +{ + int k; + Py_ssize_t outstride; + + if (nd==0) { + memcpy(dest, src, itemsize); + } + else if (nd == 1) { + for (k = 0; kndim); + if (indices == NULL) { + PyErr_NoMemory(); + return -1; + } + for (k=0; kndim;k++) { + indices[k] = 0; + } + + elements = 1; + for (k=0; kndim; k++) { + elements *= view->shape[k]; + } + if (fort == 'F') { + func = _add_one_to_index_F; + } + else { + func = _add_one_to_index_C; + } + while (elements--) { + func(view->ndim, indices, view->shape); + ptr = PyBuffer_GetPointer(view, indices); + memcpy(dest, ptr, view->itemsize); + dest += view->itemsize; + } + + PyMem_Free(indices); + return 0; +} + +/* + Get a the data from an object as a contiguous chunk of memory (in + either 'C' or 'F'ortran order) even if it means copying it into a + separate memory area. + + Returns a new reference to a Memory view object. If no copy is needed, + the memory view object points to the original memory and holds a + lock on the original. If a copy is needed, then the memory view object + points to a brand-new Bytes object (and holds a memory lock on it). + + buffertype + + PyBUF_READ buffer only needs to be read-only + PyBUF_WRITE buffer needs to be writeable (give error if not contiguous) + PyBUF_SHADOW buffer needs to be writeable so shadow it with + a contiguous buffer if it is not. The view will point to + the shadow buffer which can be written to and then + will be copied back into the other buffer when the memory + view is de-allocated. + */ + +PyObject * +PyMemoryView_GetContiguous(PyObject *obj, int buffertype, char fort) +{ + PyMemoryViewObject *mem; + PyObject *bytes; + PyBuffer *view; + int flags; + char *dest; + + if (!PyObject_CheckBuffer(obj)) { + PyErr_SetString(PyExc_TypeError, + "object does not have the buffer interface"); + return NULL; + } + + mem = PyObject_New(PyMemoryViewObject, &PyMemoryView_Type); + if (mem == NULL) return NULL; + + view = &PyMemoryView(mem); + flags = PyBUF_FULL_RO; + switch(buffertype) { + case PyBUF_WRITE: + flags = PyBUF_FULL; + break; + case PyBUF_SHADOW: + flags = PyBUF_FULL_LCK; + break; + } + + if (PyObject_GetBuffer(obj, view, flags) != 0) { + PyObject_DEL(mem); + return NULL; + } + + if (PyBuffer_IsContiguous(view, fort)) { + /* no copy needed */ + Py_INCREF(obj); + mem->base = obj; + return (PyObject *)mem; + } + /* otherwise a copy is needed */ + if (buffertype == PyBUF_WRITE) { + PyObject_DEL(mem); + PyErr_SetString(PyExc_BufferError, + "writeable contiguous buffer requested for a non-contiguous" \ + "object."); + return NULL; + } + bytes = PyBytes_FromStringAndSize(NULL, view->len); + if (bytes == NULL) { + PyObject_ReleaseBuffer(obj, view); + return NULL; + } + dest = PyBytes_AS_STRING(bytes); + /* different copying strategy depending on whether + or not any pointer de-referencing is needed + */ + /* strided or in-direct copy */ + if (view->suboffsets==NULL) { + _strided_copy_nd(dest, view->buf, view->ndim, view->shape, + view->strides, view->itemsize, fort); + } + else { + if (_indirect_copy_nd(dest, view, fort) < 0) { + Py_DECREF(bytes); + PyObject_ReleaseBuffer(obj, view); + return NULL; + } + } + if (buffertype == PyBUF_SHADOW) { + /* return a shadowed memory-view object */ + view->buf = dest; + mem->base = PyTuple_Pack(2, obj, bytes); + Py_DECREF(bytes); + } + else { + PyObject_ReleaseBuffer(obj, view); + /* steal the reference */ + mem->base = bytes; + } + return (PyObject *)mem; +} + + +static PyObject * +memory_format_get(PyMemoryViewObject *self) +{ + return PyUnicode_FromString(self->view.format); +} + +static PyObject * +memory_itemsize_get(PyMemoryViewObject *self) +{ + return PyInt_FromLong(self->view.itemsize); +} + +static PyObject * +_IntTupleFromSsizet(int len, Py_ssize_t *vals) +{ + int i; + PyObject *o; + PyObject *intTuple; + + if (vals == NULL) { + Py_INCREF(Py_None); + return Py_None; + } + intTuple = PyTuple_New(len); + if (!intTuple) return NULL; + for(i=0; iview.ndim, self->view.shape); +} + +static PyObject * +memory_strides_get(PyMemoryViewObject *self) +{ + return _IntTupleFromSsizet(self->view.ndim, self->view.strides); +} + +static PyObject * +memory_suboffsets_get(PyMemoryViewObject *self) +{ + return _IntTupleFromSsizet(self->view.ndim, self->view.suboffsets); +} + +static PyObject * +memory_size_get(PyMemoryViewObject *self) +{ + return PyInt_FromSsize_t(self->view.len); +} + +static PyObject * +memory_readonly_get(PyMemoryViewObject *self) +{ + return PyInt_FromLong(self->view.readonly); +} + +static PyObject * +memory_ndim_get(PyMemoryViewObject *self) +{ + return PyInt_FromLong(self->view.ndim); +} + +static PyGetSetDef memory_getsetlist[] ={ + {"format", + (getter)memory_format_get, + NULL, NULL}, + {"itemsize", + (getter)memory_itemsize_get, + NULL, NULL}, + {"shape", + (getter)memory_shape_get, + NULL, NULL}, + {"strides", + (getter)memory_strides_get, + NULL, NULL}, + {"suboffsets", + (getter)memory_suboffsets_get, + NULL, NULL}, + {"size", + (getter)memory_size_get, + NULL, NULL}, + {"readonly", + (getter)memory_readonly_get, + NULL, NULL}, + {"ndim", + (getter)memory_ndim_get, + NULL, NULL}, + {NULL, NULL, NULL, NULL}, +}; + + +static PyObject * +memory_tobytes(PyMemoryViewObject *mem, PyObject *args) +{ + if (!PyArg_ParseTuple(args, "")) return NULL; + /* Create new Bytes object for data */ + return PyBytes_FromObject((PyObject *)mem); +} + +static PyObject * +memory_tolist(PyMemoryViewObject *mem, PyObject *args) +{ + if (!PyArg_ParseTuple(args, "")) return NULL; + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; +} + + + +static PyMethodDef memory_methods[] = { + {"tobytes", (PyCFunction)memory_tobytes, 1, NULL}, + {"tolist", (PyCFunction)memory_tolist, 1, NULL}, + {NULL, NULL} /* sentinel */ +}; + + +static void +memory_dealloc(PyMemoryViewObject *self) +{ + + if (PyTuple_Check(self->base)) { + /* Special case when first element is generic object + with buffer interface and the second element is a + contiguous "shadow" that must be copied back into + the data areay of the first tuple element before + releasing the buffer on the first element. + */ + + PyObject_CopyData(PyTuple_GET_ITEM(self->base,0), + PyTuple_GET_ITEM(self->base,1)); + + /* The view member should have readonly == -1 in + this instance indicating that the memory can + be "locked" and was locked and will be unlocked + again after this call. + */ + PyObject_ReleaseBuffer(PyTuple_GET_ITEM(self->base,0), + &(self->view)); + } + else { + PyObject_ReleaseBuffer(self->base, &(self->view)); + } + Py_DECREF(self->base); + PyObject_DEL(self); +} + +static PyObject * +memory_repr(PyMemoryViewObject *self) +{ + + if ( self->base == NULL ) + return PyUnicode_FromFormat("", + self); + else + return PyUnicode_FromFormat( + "", + self); +} + + +static PyObject * +memory_str(PyMemoryViewObject *self) +{ + PyBuffer view; + PyObject *res; + + if (PyObject_GetBuffer((PyObject *)self, &view, PyBUF_FULL) < 0) + return NULL; + + res = PyBytes_FromStringAndSize(NULL, view.len); + PyBuffer_ToContiguous(PyBytes_AS_STRING(res), &view, view.len, 'C'); + PyObject_ReleaseBuffer((PyObject *)self, &view); + return res; +} + +/* Sequence methods */ + +static Py_ssize_t +memory_length(PyMemoryViewObject *self) +{ + PyBuffer view; + + if (PyObject_GetBuffer((PyObject *)self, &view, PyBUF_FULL) < 0) + return -1; + PyObject_ReleaseBuffer((PyObject *)self, &view); + return view.len; +} + +static PyObject * +memory_subscript(PyMemoryViewObject *self, PyObject *key) +{ + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; +} + +static int +memory_ass_sub(PyMemoryViewObject *self, PyObject *key, PyObject *value) +{ + return 0; +} + +/* As mapping */ +static PyMappingMethods memory_as_mapping = { + (lenfunc)memory_length, /*mp_length*/ + (binaryfunc)memory_subscript, /*mp_subscript*/ + (objobjargproc)memory_ass_sub, /*mp_ass_subscript*/ +}; + + +/* Buffer methods */ + +static PyBufferProcs memory_as_buffer = { + (getbufferproc)memory_getbuf, /* bf_getbuffer */ + (releasebufferproc)memory_releasebuf, /* bf_releasebuffer */ +}; + + +PyTypeObject PyMemoryView_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "memoryview", + sizeof(PyMemoryViewObject), + 0, + (destructor)memory_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + (reprfunc)memory_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + &memory_as_mapping, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + (reprfunc)memory_str, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + &memory_as_buffer, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + memory_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + memory_methods, /* tp_methods */ + 0, /* tp_members */ + memory_getsetlist, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + memory_new, /* tp_new */ +}; diff --git a/Objects/stringobject.c b/Objects/stringobject.c index b1d711d..fff4b45 100644 --- a/Objects/stringobject.c +++ b/Objects/stringobject.c @@ -1171,44 +1171,10 @@ string_subscript(PyStringObject* self, PyObject* item) } } -static Py_ssize_t -string_buffer_getreadbuf(PyStringObject *self, Py_ssize_t index, const void **ptr) -{ - if ( index != 0 ) { - PyErr_SetString(PyExc_SystemError, - "accessing non-existent string segment"); - return -1; - } - *ptr = (void *)self->ob_sval; - return Py_Size(self); -} - -static Py_ssize_t -string_buffer_getwritebuf(PyStringObject *self, Py_ssize_t index, const void **ptr) -{ - PyErr_SetString(PyExc_TypeError, - "Cannot use string as modifiable buffer"); - return -1; -} - -static Py_ssize_t -string_buffer_getsegcount(PyStringObject *self, Py_ssize_t *lenp) -{ - if ( lenp ) - *lenp = Py_Size(self); - return 1; -} - -static Py_ssize_t -string_buffer_getcharbuf(PyStringObject *self, Py_ssize_t index, const char **ptr) +static int +string_buffer_getbuffer(PyStringObject *self, PyBuffer *view, int flags) { - if ( index != 0 ) { - PyErr_SetString(PyExc_SystemError, - "accessing non-existent string segment"); - return -1; - } - *ptr = self->ob_sval; - return Py_Size(self); + return PyBuffer_FillInfo(view, (void *)self->ob_sval, Py_Size(self), 0, flags); } static PySequenceMethods string_as_sequence = { @@ -1229,14 +1195,11 @@ static PyMappingMethods string_as_mapping = { }; static PyBufferProcs string_as_buffer = { - (readbufferproc)string_buffer_getreadbuf, - (writebufferproc)string_buffer_getwritebuf, - (segcountproc)string_buffer_getsegcount, - (charbufferproc)string_buffer_getcharbuf, + (getbufferproc)string_buffer_getbuffer, + NULL, }; - #define LEFTSTRIP 0 #define RIGHTSTRIP 1 #define BOTHSTRIP 2 diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 8cc5058..f042f4a 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3251,10 +3251,8 @@ inherit_slots(PyTypeObject *type, PyTypeObject *base) basebase = base->tp_base; if (basebase->tp_as_buffer == NULL) basebase = NULL; - COPYBUF(bf_getreadbuffer); - COPYBUF(bf_getwritebuffer); - COPYBUF(bf_getsegcount); - COPYBUF(bf_getcharbuffer); + COPYBUF(bf_getbuffer); + COPYBUF(bf_releasebuffer); } basebase = base->tp_base; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 5ee3347..157ea1c 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -8108,57 +8108,26 @@ static PyMappingMethods unicode_as_mapping = { (objobjargproc)0, /* mp_ass_subscript */ }; -static Py_ssize_t -unicode_buffer_getreadbuf(PyUnicodeObject *self, - Py_ssize_t index, - const void **ptr) -{ - if (index != 0) { - PyErr_SetString(PyExc_SystemError, - "accessing non-existent unicode segment"); - return -1; - } - *ptr = (void *) self->str; - return PyUnicode_GET_DATA_SIZE(self); -} - -static Py_ssize_t -unicode_buffer_getwritebuf(PyUnicodeObject *self, Py_ssize_t index, - const void **ptr) -{ - PyErr_SetString(PyExc_TypeError, - "cannot use unicode as modifiable buffer"); - return -1; -} static int -unicode_buffer_getsegcount(PyUnicodeObject *self, - Py_ssize_t *lenp) +unicode_buffer_getbuffer(PyUnicodeObject *self, PyBuffer *view, int flags) { - if (lenp) - *lenp = PyUnicode_GET_DATA_SIZE(self); - return 1; -} - -static Py_ssize_t -unicode_buffer_getcharbuf(PyUnicodeObject *self, - Py_ssize_t index, - const void **ptr) -{ - PyObject *str; - if (index != 0) { - PyErr_SetString(PyExc_SystemError, - "accessing non-existent unicode segment"); - return -1; + if (flags & PyBUF_CHARACTER) { + PyObject *str; + + str = _PyUnicode_AsDefaultEncodedString((PyObject *)self, NULL); + if (str == NULL) return -1; + return PyBuffer_FillInfo(view, (void *)PyString_AS_STRING(str), + PyString_GET_SIZE(str), 1, flags); + } + else { + return PyBuffer_FillInfo(view, (void *)self->str, + PyUnicode_GET_DATA_SIZE(self), 1, flags); } - str = _PyUnicode_AsDefaultEncodedString((PyObject *)self, NULL); - if (str == NULL) - return -1; - *ptr = (void *) PyString_AS_STRING(str); - return PyString_GET_SIZE(str); } + /* Helpers for PyUnicode_Format() */ static PyObject * @@ -8853,10 +8822,8 @@ PyObject *PyUnicode_Format(PyObject *format, } static PyBufferProcs unicode_as_buffer = { - (readbufferproc) unicode_buffer_getreadbuf, - (writebufferproc) unicode_buffer_getwritebuf, - (segcountproc) unicode_buffer_getsegcount, - (charbufferproc) unicode_buffer_getcharbuf, + (getbufferproc) unicode_buffer_getbuffer, + NULL, }; static PyObject * diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 14fb349..ce8b7f5 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1704,6 +1704,7 @@ _PyBuiltin_Init(void) SETBUILTIN("basestring", &PyBaseString_Type); SETBUILTIN("bool", &PyBool_Type); SETBUILTIN("buffer", &PyBuffer_Type); + SETBUILTIN("memoryview", &PyMemoryView_Type); SETBUILTIN("bytes", &PyBytes_Type); SETBUILTIN("classmethod", &PyClassMethod_Type); #ifndef WITHOUT_COMPLEX diff --git a/Python/getargs.c b/Python/getargs.c index ce1fef7..a0aa872 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1179,21 +1179,31 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, void **p = va_arg(*p_va, void **); PyBufferProcs *pb = arg->ob_type->tp_as_buffer; int count; + int temp=-1; + PyBuffer view; if (pb == NULL || - pb->bf_getwritebuffer == NULL || - pb->bf_getsegcount == NULL) - return converterr("read-write buffer", arg, msgbuf, bufsize); - if ((*pb->bf_getsegcount)(arg, NULL) != 1) + pb->bf_getbuffer == NULL || + ((temp = (*pb->bf_getbuffer)(arg, &view, + PyBUF_SIMPLE)) != 0) || + view.readonly == 1) { + if (temp==0 && pb->bf_releasebuffer != NULL) { + (*pb->bf_releasebuffer)(arg, &view); + } return converterr("single-segment read-write buffer", arg, msgbuf, bufsize); - if ((count = pb->bf_getwritebuffer(arg, 0, p)) < 0) + } + + if ((count = view.len) < 0) return converterr("(unspecified)", arg, msgbuf, bufsize); + *p = view.buf; if (*format == '#') { FETCH_SIZE; STORE_SIZE(count); format++; } + if (pb->bf_releasebuffer != NULL) + (*pb->bf_releasebuffer)(arg, &view); break; } @@ -1201,23 +1211,27 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, char **p = va_arg(*p_va, char **); PyBufferProcs *pb = arg->ob_type->tp_as_buffer; int count; + PyBuffer view; if (*format++ != '#') return converterr( "invalid use of 't' format character", arg, msgbuf, bufsize); - if (pb == NULL || pb->bf_getcharbuffer == NULL || - pb->bf_getsegcount == NULL) + if (pb == NULL || pb->bf_getbuffer == NULL) return converterr( "string or read-only character buffer", arg, msgbuf, bufsize); - if (pb->bf_getsegcount(arg, NULL) != 1) - return converterr( - "string or single-segment read-only buffer", - arg, msgbuf, bufsize); + if ((*pb->bf_getbuffer)(arg, &view, PyBUF_CHARACTER) != 0) + return converterr("string or single-segment read-only buffer", + arg, msgbuf, bufsize); - count = pb->bf_getcharbuffer(arg, 0, p); + count = view.len; + *p = view.buf; + /* XXX : shouldn't really release buffer, but it should be O.K. + */ + if (pb->bf_releasebuffer != NULL) + (*pb->bf_releasebuffer)(arg, &view); if (count < 0) return converterr("(unspecified)", arg, msgbuf, bufsize); { @@ -1241,19 +1255,24 @@ convertbuffer(PyObject *arg, void **p, char **errmsg) { PyBufferProcs *pb = arg->ob_type->tp_as_buffer; Py_ssize_t count; + PyBuffer view; + + *errmsg = NULL; + *p = NULL; if (pb == NULL || - pb->bf_getreadbuffer == NULL || - pb->bf_getsegcount == NULL) { + pb->bf_getbuffer == NULL) { *errmsg = "string or read-only buffer"; return -1; } - if ((*pb->bf_getsegcount)(arg, NULL) != 1) { + + if ((*pb->bf_getbuffer)(arg, &view, PyBUF_SIMPLE) != 0) { *errmsg = "string or single-segment read-only buffer"; return -1; } - if ((count = (*pb->bf_getreadbuffer)(arg, 0, p)) < 0) { - *errmsg = "(unspecified)"; - } + count = view.len; + *p = view.buf; + if (pb->bf_releasebuffer != NULL) + (*pb->bf_releasebuffer)(arg, &view); return count; } diff --git a/Python/marshal.c b/Python/marshal.c index 975ec8d..c5952b2 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -358,12 +358,18 @@ w_object(PyObject *v, WFILE *p) w_long(co->co_firstlineno, p); w_object(co->co_lnotab, p); } - else if (PyObject_CheckReadBuffer(v)) { + else if (PyObject_CheckBuffer(v)) { /* Write unknown buffer-style objects as a string */ char *s; PyBufferProcs *pb = v->ob_type->tp_as_buffer; + PyBuffer view; + if ((*pb->bf_getbuffer)(v, &view, PyBUF_SIMPLE) != 0) { + w_byte(TYPE_UNKNOWN, p); + p->error = 1; + } w_byte(TYPE_STRING, p); - n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s); + n = view.len; + s = view.buf; if (n > INT_MAX) { p->depth--; p->error = 1; @@ -371,6 +377,8 @@ w_object(PyObject *v, WFILE *p) } w_long((long)n, p); w_string(s, (int)n, p); + if (pb->bf_releasebuffer != NULL) + (*pb->bf_releasebuffer)(v, &view); } else { w_byte(TYPE_UNKNOWN, p); -- cgit v0.12