Backport from py3k: Implement the new buffer interface from pep3118

for ctypes instances.  Closes issue #2404.

4 files changed, 264 insertions, 24 deletions

diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c
index 740b7f6..30e981a 100644
--- a/Modules/_ctypes/_ctypes.c
+++ b/Modules/_ctypes/_ctypes.c
@@ -294,6 +294,36 @@ PyDict_GetItemProxy(PyObject *dict, PyObject *key)
 
 /******************************************************************/
 /*
+  Allocate a memory block for a pep3118 format string, copy prefix (if
+  non-null) and suffix into it.  Returns NULL on failure, with the error
+  indicator set.  If called with a suffix of NULL the error indicator must
+  already be set.
+ */
+char *
+alloc_format_string(const char *prefix, const char *suffix)
+{
+	size_t len;
+	char *result;
+
+	if (suffix == NULL) {
+		assert(PyErr_Occurred());
+		return NULL;
+	}
+	len = strlen(suffix);
+	if (prefix)
+		len += strlen(prefix);
+	result = PyMem_Malloc(len + 1);
+	if (result == NULL)
+		return NULL;
+	if (prefix)
+		strcpy(result, prefix);
+	else
+		result[0] = '\0';
+	strcat(result, suffix);
+	return result;
+}
+
+/*
   StructType_Type - a meta type/class.  Creating a new class using this one as
   __metaclass__ will call the contructor StructUnionType_new.  It replaces the
   tp_dict member with a new instance of StgDict, and initializes the C
@@ -874,6 +904,16 @@ PointerType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 		return NULL;
 	}
 
+	if (proto) {
+		StgDictObject *itemdict = PyType_stgdict(proto);
+		assert(itemdict);
+		stgdict->format = alloc_format_string("&", itemdict->format);
+		if (stgdict->format == NULL) {
+			Py_DECREF((PyObject *)stgdict);
+			return NULL;
+		}
+	}
+
 	/* create the new instance (which is a class,
 	   since we are a metatype!) */
 	result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds);
@@ -1244,9 +1284,10 @@ ArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 	StgDictObject *itemdict;
 	PyObject *proto;
 	PyObject *typedict;
-	int length;
+	long length;
 
 	Py_ssize_t itemsize, itemalign;
+	char buf[32];
 
 	typedict = PyTuple_GetItem(args, 2);
 	if (!typedict)
@@ -1281,6 +1322,28 @@ ArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 		return NULL;
 	}
 
+	assert(itemdict->format);
+	if (itemdict->format[0] == '(') {
+		sprintf(buf, "(%ld,", length);
+		stgdict->format = alloc_format_string(buf, itemdict->format+1);
+	} else {
+		sprintf(buf, "(%ld)", length);
+		stgdict->format = alloc_format_string(buf, itemdict->format);
+	}
+	if (stgdict->format == NULL) {
+		Py_DECREF((PyObject *)stgdict);
+		return NULL;
+	}
+	stgdict->ndim = itemdict->ndim + 1;
+	stgdict->shape = PyMem_Malloc(sizeof(Py_ssize_t *) * stgdict->ndim);
+	if (stgdict->shape == NULL) {
+		Py_DECREF((PyObject *)stgdict);
+		return NULL;
+	}
+	stgdict->shape[0] = length;
+	memmove(&stgdict->shape[1], itemdict->shape,
+		sizeof(Py_ssize_t) * (stgdict->ndim - 1));
+
 	itemsize = itemdict->size;
 	if (length * itemsize < 0) {
 		PyErr_SetString(PyExc_OverflowError,
@@ -1768,6 +1831,8 @@ SimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 	PyTypeObject *result;
 	StgDictObject *stgdict;
 	PyObject *proto;
+	const char *proto_str;
+	Py_ssize_t proto_len;
 	PyMethodDef *ml;
 	struct fielddesc *fmt;
 
@@ -1778,17 +1843,34 @@ SimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 		return NULL;
 
 	proto = PyObject_GetAttrString((PyObject *)result, "_type_"); /* new ref */
-	if (!proto
-	    || !PyBytes_Check(proto)
-	    || 1 != strlen(PyBytes_AS_STRING(proto))
-	    || !strchr(SIMPLE_TYPE_CHARS, PyBytes_AS_STRING(proto)[0])) {
+	if (!proto) {
+		PyErr_SetString(PyExc_AttributeError,
+				"class must define a '_type_' attribute");
+  error:
+		Py_XDECREF(proto);
+		Py_XDECREF(result);
+		return NULL;
+	}
+	if (PyString_Check(proto)) {
+		proto_str = PyBytes_AS_STRING(proto);
+		proto_len = PyBytes_GET_SIZE(proto);
+	} else {
+		PyErr_SetString(PyExc_TypeError,
+			"class must define a '_type_' string attribute");
+		goto error;
+	}
+	if (proto_len != 1) {
+		PyErr_SetString(PyExc_ValueError,
+				"class must define a '_type_' attribute "
+				"which must be a string of length 1");
+		goto error;
+	}
+	if (!strchr(SIMPLE_TYPE_CHARS, *proto_str)) {
 		PyErr_Format(PyExc_AttributeError,
 			     "class must define a '_type_' attribute which must be\n"
 			     "a single character string containing one of '%s'.",
 			     SIMPLE_TYPE_CHARS);
-		Py_XDECREF(proto);
-		Py_DECREF(result);
-		return NULL;
+		goto error;
 	}
 	fmt = getentry(PyBytes_AS_STRING(proto));
 	if (fmt == NULL) {
@@ -1810,6 +1892,16 @@ SimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 	stgdict->size = fmt->pffi_type->size;
 	stgdict->setfunc = fmt->setfunc;
 	stgdict->getfunc = fmt->getfunc;
+#ifdef WORDS_BIGENDIAN
+	stgdict->format = alloc_format_string(">", proto_str);
+#else
+	stgdict->format = alloc_format_string("<", proto_str);
+#endif
+	if (stgdict->format == NULL) {
+		Py_DECREF(result);
+		Py_DECREF((PyObject *)stgdict);
+		return NULL;
+	}
 
 	stgdict->paramfunc = SimpleType_paramfunc;
 /*
@@ -1895,22 +1987,32 @@ SimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 	if (type == &SimpleType_Type && fmt->setfunc_swapped && fmt->getfunc_swapped) {
 		PyObject *swapped = CreateSwappedType(type, args, kwds,
 						      proto, fmt);
+		StgDictObject *sw_dict;
 		if (swapped == NULL) {
 			Py_DECREF(result);
 			return NULL;
 		}
+		sw_dict = PyType_stgdict(swapped);
 #ifdef WORDS_BIGENDIAN
 		PyObject_SetAttrString((PyObject *)result, "__ctype_le__", swapped);
 		PyObject_SetAttrString((PyObject *)result, "__ctype_be__", (PyObject *)result);
 		PyObject_SetAttrString(swapped, "__ctype_be__", (PyObject *)result);
 		PyObject_SetAttrString(swapped, "__ctype_le__", swapped);
+		/* We are creating the type for the OTHER endian */
+		sw_dict->format = alloc_format_string("<", stgdict->format+1);
 #else
 		PyObject_SetAttrString((PyObject *)result, "__ctype_be__", swapped);
 		PyObject_SetAttrString((PyObject *)result, "__ctype_le__", (PyObject *)result);
 		PyObject_SetAttrString(swapped, "__ctype_le__", (PyObject *)result);
 		PyObject_SetAttrString(swapped, "__ctype_be__", swapped);
+		/* We are creating the type for the OTHER endian */
+		sw_dict->format = alloc_format_string(">", stgdict->format+1);
 #endif
 		Py_DECREF(swapped);
+		if (PyErr_Occurred()) {
+			Py_DECREF(result);
+			return NULL;
+		}
 	};
 
 	return (PyObject *)result;
@@ -2166,6 +2268,13 @@ CFuncPtrType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 		return NULL;
 
 	stgdict->paramfunc = CFuncPtrType_paramfunc;
+	/* We do NOT expose the function signature in the format string.  It
+	   is impossible, generally, because the only requirement for the
+	   argtypes items is that they have a .from_param method - we do not
+	   know the types of the arguments (although, in practice, most
+	   argtypes would be a ctypes type).
+	*/
+	stgdict->format = alloc_format_string(NULL, "X{}");
 	stgdict->flags |= TYPEFLAG_ISPOINTER;
 
 	/* create the new instance (which is a class,
@@ -2386,15 +2495,34 @@ static PyMemberDef CData_members[] = {
 	{ NULL },
 };
 
-static Py_ssize_t CData_GetBuffer(PyObject *_self, Py_ssize_t seg, void **pptr)
+static int CData_NewGetBuffer(PyObject *_self, Py_buffer *view, int flags)
 {
 	CDataObject *self = (CDataObject *)_self;
-	if (seg != 0) {
-		/* Hm. Must this set an exception? */
+	StgDictObject *dict = PyObject_stgdict(_self);
+	Py_ssize_t i;
+
+	if (view == NULL) return 0;
+	if (((flags & PyBUF_LOCK) == PyBUF_LOCK)) {
+		PyErr_SetString(PyExc_BufferError,
+				"Cannot lock this object.");
 		return -1;
 	}
-	*pptr = self->b_ptr;
-	return self->b_size;
+
+	view->buf = self->b_ptr;
+	view->len = self->b_size;
+	view->readonly = 0;
+	/* use default format character if not set */
+	view->format = dict->format ? dict->format : "B";
+	view->ndim = dict->ndim;
+	view->shape = dict->shape;
+	view->itemsize = self->b_size;
+	for (i = 0; i < view->ndim; ++i) {
+		view->itemsize /= dict->shape[i];
+	}
+	view->strides = NULL;
+	view->suboffsets = NULL;
+	view->internal = NULL;
+	return 0;
 }
 
 static Py_ssize_t CData_GetSegcount(PyObject *_self, Py_ssize_t *lenp)
@@ -2404,11 +2532,24 @@ static Py_ssize_t CData_GetSegcount(PyObject *_self, Py_ssize_t *lenp)
 	return 1;
 }
 
+static Py_ssize_t CData_GetBuffer(PyObject *_self, Py_ssize_t seg, void **pptr)
+{
+	CDataObject *self = (CDataObject *)_self;
+	if (seg != 0) {
+		/* Hm. Must this set an exception? */
+		return -1;
+	}
+	*pptr = self->b_ptr;
+	return self->b_size;
+}
+
 static PyBufferProcs CData_as_buffer = {
-	CData_GetBuffer,
-	CData_GetBuffer,
-	CData_GetSegcount,
-	NULL,
+	(readbufferproc)CData_GetBuffer,
+	(writebufferproc)CData_GetBuffer,
+	(segcountproc)CData_GetSegcount,
+	(charbufferproc)NULL,
+	(getbufferproc)CData_NewGetBuffer,
+	(releasebufferproc)NULL,
 };
 
 /*
@@ -2497,7 +2638,7 @@ PyTypeObject CData_Type = {
 	0,					/* tp_getattro */
 	0,					/* tp_setattro */
 	&CData_as_buffer,			/* tp_as_buffer */
-	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
+	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_NEWBUFFER | Py_TPFLAGS_BASETYPE, /* tp_flags */
 	"XXX to be provided",			/* tp_doc */
 	(traverseproc)CData_traverse,		/* tp_traverse */
 	(inquiry)CData_clear,			/* tp_clear */
@@ -3824,7 +3965,7 @@ PyTypeObject CFuncPtr_Type = {
 	0,					/* tp_getattro */
 	0,					/* tp_setattro */
 	&CData_as_buffer,			/* tp_as_buffer */
-	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
+	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_NEWBUFFER | Py_TPFLAGS_BASETYPE, /* tp_flags */
 	"Function Pointer",			/* tp_doc */
 	(traverseproc)CFuncPtr_traverse,	/* tp_traverse */
 	(inquiry)CFuncPtr_clear,		/* tp_clear */
@@ -3967,7 +4108,7 @@ static PyTypeObject Struct_Type = {
 	0,					/* tp_getattro */
 	0,					/* tp_setattro */
 	&CData_as_buffer,			/* tp_as_buffer */
-	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
+	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_NEWBUFFER | Py_TPFLAGS_BASETYPE, /* tp_flags */
 	"Structure base class",			/* tp_doc */
 	(traverseproc)CData_traverse,		/* tp_traverse */
 	(inquiry)CData_clear,			/* tp_clear */
@@ -4009,7 +4150,7 @@ static PyTypeObject Union_Type = {
 	0,					/* tp_getattro */
 	0,					/* tp_setattro */
 	&CData_as_buffer,			/* tp_as_buffer */
-	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
+	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_NEWBUFFER | Py_TPFLAGS_BASETYPE, /* tp_flags */
 	"Union base class",			/* tp_doc */
 	(traverseproc)CData_traverse,		/* tp_traverse */
 	(inquiry)CData_clear,			/* tp_clear */
@@ -4406,7 +4547,7 @@ PyTypeObject Array_Type = {
 	0,					/* tp_getattro */
 	0,					/* tp_setattro */
 	&CData_as_buffer,			/* tp_as_buffer */
-	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
+	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_NEWBUFFER | Py_TPFLAGS_BASETYPE, /* tp_flags */
 	"XXX to be provided",			/* tp_doc */
 	(traverseproc)CData_traverse,		/* tp_traverse */
 	(inquiry)CData_clear,			/* tp_clear */
@@ -4643,7 +4784,7 @@ static PyTypeObject Simple_Type = {
 	0,					/* tp_getattro */
 	0,					/* tp_setattro */
 	&CData_as_buffer,			/* tp_as_buffer */
-	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
+	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_NEWBUFFER | Py_TPFLAGS_BASETYPE, /* tp_flags */
 	"XXX to be provided",			/* tp_doc */
 	(traverseproc)CData_traverse,		/* tp_traverse */
 	(inquiry)CData_clear,			/* tp_clear */
@@ -5043,7 +5184,7 @@ PyTypeObject Pointer_Type = {
 	0,					/* tp_getattro */
 	0,					/* tp_setattro */
 	&CData_as_buffer,			/* tp_as_buffer */
-	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
+	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_NEWBUFFER | Py_TPFLAGS_BASETYPE, /* tp_flags */
 	"XXX to be provided",			/* tp_doc */
 	(traverseproc)CData_traverse,		/* tp_traverse */
 	(inquiry)CData_clear,			/* tp_clear */
diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c
index 95b2709..0520176 100644
--- a/Modules/_ctypes/callproc.c
+++ b/Modules/_ctypes/callproc.c
@@ -1666,10 +1666,37 @@ pointer(PyObject *self, PyObject *arg)
 	return result;
 }
 
+static PyObject *
+buffer_info(PyObject *self, PyObject *arg)
+{
+	StgDictObject *dict = PyType_stgdict(arg);
+	PyObject *shape;
+	Py_ssize_t i;
+
+	if (dict == NULL)
+		dict = PyObject_stgdict(arg);
+	if (dict == NULL) {
+		PyErr_SetString(PyExc_TypeError,
+				"not a ctypes type or object");
+		return NULL;
+	}
+	shape = PyTuple_New(dict->ndim);
+	for (i = 0; i < (int)dict->ndim; ++i)
+		PyTuple_SET_ITEM(shape, i, PyLong_FromSsize_t(dict->shape[i]));
+
+	if (PyErr_Occurred()) {
+		Py_DECREF(shape);
+		return NULL;
+	}
+	return Py_BuildValue("siN", dict->format, dict->ndim, shape);
+}
+
 PyMethodDef module_methods[] = {
 	{"POINTER", POINTER, METH_O },
 	{"pointer", pointer, METH_O },
 	{"_unpickle", unpickle, METH_VARARGS },
+	{"_buffer_info", buffer_info, METH_O,
+	 "Return buffer interface information (for testing only)"},
 	{"resize", resize, METH_VARARGS, "Resize the memory buffer of a ctypes instance"},
 #ifdef CTYPES_UNICODE
 	{"set_conversion_mode", set_conversion_mode, METH_VARARGS, set_conversion_mode_doc},
diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h
index 1a104cf..d068ea5 100644
--- a/Modules/_ctypes/ctypes.h
+++ b/Modules/_ctypes/ctypes.h
@@ -235,6 +235,14 @@ typedef struct {
 	PyObject *restype;	/* CDataObject or NULL */
 	PyObject *checker;
 	int flags;		/* calling convention and such */
+
+	/* pep3118 fields, pointers neeed PyMem_Free */
+	char *format;
+	int ndim;
+	Py_ssize_t *shape;
+/*	Py_ssize_t *strides;	*/ /* unused in ctypes */
+/*	Py_ssize_t *suboffsets;	*/ /* unused in ctypes */
+
 } StgDictObject;
 
 /****************************************************************
@@ -415,6 +423,7 @@ extern void *MallocClosure(void);
 extern void _AddTraceback(char *, char *, int);
 
 extern PyObject *CData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr);
+extern char *alloc_format_string(const char *prefix, const char *suffix);
 
 /* XXX better name needed! */
 extern int IsSimpleSubType(PyObject *obj);
diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c
index f7f19cf..cd38ecd 100644
--- a/Modules/_ctypes/stgdict.c
+++ b/Modules/_ctypes/stgdict.c
@@ -6,6 +6,7 @@
 #include <ffi.h>
 #ifdef MS_WIN32
 #include <windows.h>
+#include <malloc.h>
 #endif
 #include "ctypes.h"
 
@@ -24,6 +25,9 @@ StgDict_init(StgDictObject *self, PyObject *args, PyObject *kwds)
 {
 	if (PyDict_Type.tp_init((PyObject *)self, args, kwds) < 0)
 		return -1;
+	self->format = NULL;
+	self->ndim = 0;
+	self->shape = NULL;
 	return 0;
 }
 
@@ -42,6 +46,8 @@ static void
 StgDict_dealloc(StgDictObject *self)
 {
 	StgDict_clear(self);
+	PyMem_Free(self->format);
+	PyMem_Free(self->shape);
 	PyMem_Free(self->ffi_type_pointer.elements);
 	PyDict_Type.tp_dealloc((PyObject *)self);
 }
@@ -54,6 +60,10 @@ StgDict_clone(StgDictObject *dst, StgDictObject *src)
 
 	StgDict_clear(dst);
 	PyMem_Free(dst->ffi_type_pointer.elements);
+	PyMem_Free(dst->format);
+	dst->format = NULL;
+	PyMem_Free(dst->shape);
+	dst->shape = NULL;
 	dst->ffi_type_pointer.elements = NULL;
 
 	d = (char *)dst;
@@ -68,6 +78,20 @@ StgDict_clone(StgDictObject *dst, StgDictObject *src)
 	Py_XINCREF(dst->restype);
 	Py_XINCREF(dst->checker);
 
+	if (src->format) {
+		dst->format = PyMem_Malloc(strlen(src->format) + 1);
+		if (dst->format == NULL)
+			return -1;
+		strcpy(dst->format, src->format);
+	}
+	if (src->shape) {
+		dst->shape = PyMem_Malloc(sizeof(Py_ssize_t) * src->ndim);
+		if (dst->shape == NULL)
+			return -1;
+		memcpy(dst->shape, src->shape,
+		       sizeof(Py_ssize_t) * src->ndim);
+	}
+
 	if (src->ffi_type_pointer.elements == NULL)
 		return 0;
 	size = sizeof(ffi_type *) * (src->length + 1);
@@ -349,6 +373,11 @@ StructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct)
 		return -1;
 	}
 
+	if (stgdict->format) {
+		PyMem_Free(stgdict->format);
+		stgdict->format = NULL;
+	}
+
 	if (stgdict->ffi_type_pointer.elements)
 		PyMem_Free(stgdict->ffi_type_pointer.elements);
 
@@ -387,6 +416,15 @@ StructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct)
 		ffi_ofs = 0;
 	}
 
+	if (isStruct && !isPacked) {
+		stgdict->format = alloc_format_string(NULL, "T{");
+	} else {
+		/* PEP3118 doesn't support union, or packed structures (well,
+		   only standard packing, but we dont support the pep for
+		   that). Use 'B' for bytes. */
+		stgdict->format = alloc_format_string(NULL, "B");
+	}
+
 #define realdict ((PyObject *)&stgdict->dict)
 	for (i = 0; i < len; ++i) {
 		PyObject *name = NULL, *desc = NULL;
@@ -451,6 +489,24 @@ StructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct)
 			}
 		} else
 			bitsize = 0;
+		if (isStruct && !isPacked) {
+			char *fieldfmt = dict->format ? dict->format : "B";
+			char *fieldname = PyString_AsString(name);
+			char *ptr;
+			Py_ssize_t len = strlen(fieldname) + strlen(fieldfmt);
+			char *buf = alloca(len + 2 + 1);
+
+			sprintf(buf, "%s:%s:", fieldfmt, fieldname);
+
+			ptr = stgdict->format;
+			stgdict->format = alloc_format_string(stgdict->format, buf);
+			PyMem_Free(ptr);
+
+			if (stgdict->format == NULL) {
+				Py_DECREF(pair);
+				return -1;
+			}
+		}
 		if (isStruct) {
 			prop = CField_FromDesc(desc, i,
 					       &field_size, bitsize, &bitofs,
@@ -481,6 +537,13 @@ StructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct)
 		Py_DECREF(prop);
 	}
 #undef realdict
+
+	if (isStruct && !isPacked) {
+		stgdict->format = alloc_format_string(stgdict->format, "}");
+		if (stgdict->format == NULL)
+			return -1;
+	}
+
 	if (!isStruct)
 		size = union_size;