bpo-29753: fix merging packed bitfields in ctypes struct/union (GH-19850)

From the commit message:

> When the structure is packed we should always expand when needed,
> otherwise we will add some padding between the fields. This patch makes
> sure we always merge bitfields together. It also changes the field merging
> algorithm so that it handles bitfields correctly.

Automerge-Triggered-By: GH:jaraco

1 files changed, 27 insertions, 8 deletions

diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c
index 5bd96f1..d96a1b0 100644
--- a/Modules/_ctypes/cfield.c
+++ b/Modules/_ctypes/cfield.c
@@ -71,6 +71,18 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index,
         Py_DECREF(self);
         return NULL;
     }
+
+#ifndef MS_WIN32
+    /* if we have a packed bitfield, calculate the minimum number of bytes we
+    need to fit it. otherwise use the specified size. */
+    if (pack && bitsize) {
+        size = (bitsize - 1) / 8 + 1;
+    } else
+#endif
+        size = dict->size;
+
+    proto = desc;
+
     if (bitsize /* this is a bitfield request */
         && *pfield_size /* we have a bitfield open */
 #ifdef MS_WIN32
@@ -87,7 +99,9 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index,
     } else if (bitsize /* this is a bitfield request */
         && *pfield_size /* we have a bitfield open */
         && dict->size * 8 >= *pfield_size
-        && (*pbitofs + bitsize) <= dict->size * 8) {
+        /* if this is a packed bitfield, always expand it.
+           otherwise calculate if we need to expand it. */
+        && (((*pbitofs + bitsize) <= dict->size * 8) || pack)) {
         /* expand bit field */
         fieldtype = EXPAND_BITFIELD;
 #endif
@@ -95,7 +109,9 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index,
         /* start new bitfield */
         fieldtype = NEW_BITFIELD;
         *pbitofs = 0;
-        *pfield_size = dict->size * 8;
+        /* use our calculated size (size) instead of type size (dict->size),
+           which can be different for packed bitfields */
+        *pfield_size = size * 8;
     } else {
         /* not a bit field */
         fieldtype = NO_BITFIELD;
@@ -103,9 +119,6 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index,
         *pfield_size = 0;
     }
 
-    size = dict->size;
-    proto = desc;
-
     /*  Field descriptors for 'c_char * n' are be scpecial cased to
         return a Python string instead of an Array object instance...
     */
@@ -170,10 +183,16 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index,
         break;
 
     case EXPAND_BITFIELD:
-        *poffset += dict->size - *pfield_size/8;
-        *psize += dict->size - *pfield_size/8;
+        /* increase the size if it is a packed bitfield.
+           EXPAND_BITFIELD should not be selected for non-packed fields if the
+           current size isn't already enough. */
+        if (pack)
+            size = (*pbitofs + bitsize - 1) / 8 + 1;
+
+        *poffset += size - *pfield_size/8;
+        *psize += size - *pfield_size/8;
 
-        *pfield_size = dict->size * 8;
+        *pfield_size = size * 8;
 
         if (big_endian)
             self->size = (bitsize << 16) + *pfield_size - *pbitofs - bitsize;