summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorGregory P. Smith <greg@krypto.org>2018-04-20 05:41:19 (GMT)
committerGitHub <noreply@github.com>2018-04-20 05:41:19 (GMT)
commit397f1b28c4a12e3b3ed59a89599eabc457412649 (patch)
tree7e95dd2ff8c41c8a84ecea8eed68215b20734ce7
parentb87c1c92fc93c5733cd3d8606ab2301ca6ba208f (diff)
downloadcpython-397f1b28c4a12e3b3ed59a89599eabc457412649.zip
cpython-397f1b28c4a12e3b3ed59a89599eabc457412649.tar.gz
cpython-397f1b28c4a12e3b3ed59a89599eabc457412649.tar.bz2
bpo-33312: Fix clang ubsan out of bounds warnings in dict. (GH-6537)
Fix clang ubsan (undefined behavior sanitizer) warnings in dictobject.c by adjusting how the internal struct _dictkeysobject shared keys structure is declared. This remains ABI compatible. We get rid of the union at the end of the struct being used for conveinence to avoid typecasting in favor of char[] variable length array at the end of a struct. This is known to clang to be used for variable sized objects and will not cause an undefined behavior problem. Similarly, char arrays do not have strict aliasing undefined behavior when cast. PEP-007 does not currently list variable length arrays (VLAs) as allowed in our subset of C99. If this turns out to be a problem, the fix to this is to change the char `dk_indices[]` into `dk_indices[1]` and restore the three size computation subtractions this change removes: `- Py_MEMBER_SIZE(PyDictKeysObject, dk_indices)` If this works as is I'll make a separate PR to update PEP-007.
-rw-r--r--Misc/NEWS.d/next/Core and Builtins/2018-04-19-08-30-07.bpo-33312.mDe2iL.rst3
-rw-r--r--Objects/dict-common.h11
-rw-r--r--Objects/dictobject.c27
3 files changed, 17 insertions, 24 deletions
diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-04-19-08-30-07.bpo-33312.mDe2iL.rst b/Misc/NEWS.d/next/Core and Builtins/2018-04-19-08-30-07.bpo-33312.mDe2iL.rst
new file mode 100644
index 0000000..40b51b9
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2018-04-19-08-30-07.bpo-33312.mDe2iL.rst
@@ -0,0 +1,3 @@
+Fixed clang ubsan (undefined behavior sanitizer) warnings in dictobject.c by
+adjusting how the internal struct _dictkeysobject shared keys structure is
+declared.
diff --git a/Objects/dict-common.h b/Objects/dict-common.h
index 3e52468..71d6b02 100644
--- a/Objects/dict-common.h
+++ b/Objects/dict-common.h
@@ -58,15 +58,8 @@ struct _dictkeysobject {
- 4 bytes if dk_size <= 0xffffffff (int32_t*)
- 8 bytes otherwise (int64_t*)
- Dynamically sized, 8 is minimum. */
- union {
- int8_t as_1[8];
- int16_t as_2[4];
- int32_t as_4[2];
-#if SIZEOF_VOID_P > 4
- int64_t as_8[1];
-#endif
- } dk_indices;
+ Dynamically sized, SIZEOF_VOID_P is minimum. */
+ char dk_indices[]; /* char is required to avoid strict aliasing. */
/* "PyDictKeyEntry dk_entries[dk_usable];" array follows:
see the DK_ENTRIES() macro */
diff --git a/Objects/dictobject.c b/Objects/dictobject.c
index 01d913b..acf757c 100644
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -298,7 +298,7 @@ PyDict_Fini(void)
2 : sizeof(int32_t))
#endif
#define DK_ENTRIES(dk) \
- ((PyDictKeyEntry*)(&(dk)->dk_indices.as_1[DK_SIZE(dk) * DK_IXSIZE(dk)]))
+ ((PyDictKeyEntry*)(&((int8_t*)((dk)->dk_indices))[DK_SIZE(dk) * DK_IXSIZE(dk)]))
#define DK_DEBUG_INCREF _Py_INC_REFTOTAL _Py_REF_DEBUG_COMMA
#define DK_DEBUG_DECREF _Py_DEC_REFTOTAL _Py_REF_DEBUG_COMMA
@@ -316,21 +316,21 @@ dk_get_index(PyDictKeysObject *keys, Py_ssize_t i)
Py_ssize_t ix;
if (s <= 0xff) {
- int8_t *indices = keys->dk_indices.as_1;
+ int8_t *indices = (int8_t*)(keys->dk_indices);
ix = indices[i];
}
else if (s <= 0xffff) {
- int16_t *indices = keys->dk_indices.as_2;
+ int16_t *indices = (int16_t*)(keys->dk_indices);
ix = indices[i];
}
#if SIZEOF_VOID_P > 4
else if (s > 0xffffffff) {
- int64_t *indices = keys->dk_indices.as_8;
+ int64_t *indices = (int64_t*)(keys->dk_indices);
ix = indices[i];
}
#endif
else {
- int32_t *indices = keys->dk_indices.as_4;
+ int32_t *indices = (int32_t*)(keys->dk_indices);
ix = indices[i];
}
assert(ix >= DKIX_DUMMY);
@@ -346,23 +346,23 @@ dk_set_index(PyDictKeysObject *keys, Py_ssize_t i, Py_ssize_t ix)
assert(ix >= DKIX_DUMMY);
if (s <= 0xff) {
- int8_t *indices = keys->dk_indices.as_1;
+ int8_t *indices = (int8_t*)(keys->dk_indices);
assert(ix <= 0x7f);
indices[i] = (char)ix;
}
else if (s <= 0xffff) {
- int16_t *indices = keys->dk_indices.as_2;
+ int16_t *indices = (int16_t*)(keys->dk_indices);
assert(ix <= 0x7fff);
indices[i] = (int16_t)ix;
}
#if SIZEOF_VOID_P > 4
else if (s > 0xffffffff) {
- int64_t *indices = keys->dk_indices.as_8;
+ int64_t *indices = (int64_t*)(keys->dk_indices);
indices[i] = ix;
}
#endif
else {
- int32_t *indices = keys->dk_indices.as_4;
+ int32_t *indices = (int32_t*)(keys->dk_indices);
assert(ix <= 0x7fffffff);
indices[i] = (int32_t)ix;
}
@@ -421,8 +421,8 @@ static PyDictKeysObject empty_keys_struct = {
lookdict_split, /* dk_lookup */
0, /* dk_usable (immutable) */
0, /* dk_nentries */
- .dk_indices = { .as_1 = {DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY,
- DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY}},
+ {DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY,
+ DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY}, /* dk_indices */
};
static PyObject *empty_values[1] = { NULL };
@@ -530,7 +530,6 @@ static PyDictKeysObject *new_keys_object(Py_ssize_t size)
}
else {
dk = PyObject_MALLOC(sizeof(PyDictKeysObject)
- - Py_MEMBER_SIZE(PyDictKeysObject, dk_indices)
+ es * size
+ sizeof(PyDictKeyEntry) * usable);
if (dk == NULL) {
@@ -543,7 +542,7 @@ static PyDictKeysObject *new_keys_object(Py_ssize_t size)
dk->dk_usable = usable;
dk->dk_lookup = lookdict_unicode_nodummy;
dk->dk_nentries = 0;
- memset(&dk->dk_indices.as_1[0], 0xff, es * size);
+ memset(&dk->dk_indices[0], 0xff, es * size);
memset(DK_ENTRIES(dk), 0, sizeof(PyDictKeyEntry) * usable);
return dk;
}
@@ -3007,7 +3006,6 @@ _PyDict_SizeOf(PyDictObject *mp)
in the type object. */
if (mp->ma_keys->dk_refcnt == 1)
res += (sizeof(PyDictKeysObject)
- - Py_MEMBER_SIZE(PyDictKeysObject, dk_indices)
+ DK_IXSIZE(mp->ma_keys) * size
+ sizeof(PyDictKeyEntry) * usable);
return res;
@@ -3017,7 +3015,6 @@ Py_ssize_t
_PyDict_KeysSize(PyDictKeysObject *keys)
{
return (sizeof(PyDictKeysObject)
- - Py_MEMBER_SIZE(PyDictKeysObject, dk_indices)
+ DK_IXSIZE(keys) * DK_SIZE(keys)
+ USABLE_FRACTION(DK_SIZE(keys)) * sizeof(PyDictKeyEntry));
}