From ca8aa4acf6755dd012706e1e38fb737ae83ab5c6 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 20 Sep 2012 20:56:47 +0200 Subject: Issue #15144: Fix possible integer overflow when handling pointers as integer values, by using Py_uintptr_t instead of size_t. Patch by Serhiy Storchaka. --- Include/objimpl.h | 9 +++------ Include/pymacro.h | 14 ++++++++++++++ Misc/NEWS | 4 ++++ Modules/_elementtree.c | 2 +- Objects/obmalloc.c | 7 ++----- Objects/stringlib/codecs.h | 14 +++++--------- Objects/stringlib/fastsearch.h | 3 +-- Objects/stringlib/find_max_char.h | 11 ++++------- Objects/unicodeobject.c | 15 ++++++--------- Parser/node.c | 2 +- Python/pyarena.c | 8 +++----- 11 files changed, 44 insertions(+), 45 deletions(-) diff --git a/Include/objimpl.h b/Include/objimpl.h index b1a624c..3d5f509 100644 --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -181,12 +181,9 @@ PyAPI_FUNC(PyVarObject *) _PyObject_NewVar(PyTypeObject *, Py_ssize_t); #endif #define _PyObject_VAR_SIZE(typeobj, nitems) \ - (size_t) \ - ( ( (typeobj)->tp_basicsize + \ - (nitems)*(typeobj)->tp_itemsize + \ - (SIZEOF_VOID_P - 1) \ - ) & ~(SIZEOF_VOID_P - 1) \ - ) + _Py_SIZE_ROUND_UP((typeobj)->tp_basicsize + \ + (nitems)*(typeobj)->tp_itemsize, \ + SIZEOF_VOID_P) #define PyObject_NEW(type, typeobj) \ ( (type *) PyObject_Init( \ diff --git a/Include/pymacro.h b/Include/pymacro.h index 1dc0c61..ce1cbef 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -52,4 +52,18 @@ #define PyDoc_STR(str) "" #endif +/* Below "a" is a power of 2. */ +/* Round down size "n" to be a multiple of "a". */ +#define _Py_SIZE_ROUND_DOWN(n, a) ((size_t)(n) & ~(size_t)((a) - 1)) +/* Round up size "n" to be a multiple of "a". */ +#define _Py_SIZE_ROUND_UP(n, a) (((size_t)(n) + \ + (size_t)((a) - 1)) & ~(size_t)((a) - 1)) +/* Round pointer "p" down to the closest "a"-aligned address <= "p". */ +#define _Py_ALIGN_DOWN(p, a) ((void *)((Py_uintptr_t)(p) & ~(Py_uintptr_t)((a) - 1))) +/* Round pointer "p" up to the closest "a"-aligned address >= "p". */ +#define _Py_ALIGN_UP(p, a) ((void *)(((Py_uintptr_t)(p) + \ + (Py_uintptr_t)((a) - 1)) & ~(Py_uintptr_t)((a) - 1))) +/* Check if pointer "p" is aligned to "a"-bytes boundary. */ +#define _Py_IS_ALIGNED(p, a) (!((Py_uintptr_t)(p) & (Py_uintptr_t)((a) - 1))) + #endif /* Py_PYMACRO_H */ diff --git a/Misc/NEWS b/Misc/NEWS index da84c3a..04580d7 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ What's New in Python 3.3.1 Core and Builtins ----------------- +- Issue #15144: Fix possible integer overflow when handling pointers as + integer values, by using Py_uintptr_t instead of size_t. Patch by + Serhiy Storchaka. + - Issue #15965: Explicitly cast AT_FDCWD as (int). Required on Solaris 10 (which defines AT_FDCWD as 0xffd19553), harmless on other platforms. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 6f17d80..43f9d9b 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -98,7 +98,7 @@ do { memory -= size; printf("%8d - %s\n", memory, comment); } while (0) info. */ #define JOIN_GET(p) ((Py_uintptr_t) (p) & 1) #define JOIN_SET(p, flag) ((void*) ((Py_uintptr_t) (JOIN_OBJ(p)) | (flag))) -#define JOIN_OBJ(p) ((PyObject*) ((Py_uintptr_t) (p) & ~1)) +#define JOIN_OBJ(p) ((PyObject*) ((Py_uintptr_t) (p) & ~(Py_uintptr_t)1)) /* glue functions (see the init function for details) */ static PyObject* elementtree_parseerror_obj; diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 9254821..6225ebb 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -138,7 +138,6 @@ static int running_on_valgrind = -1; */ #define ALIGNMENT 8 /* must be 2^N */ #define ALIGNMENT_SHIFT 3 -#define ALIGNMENT_MASK (ALIGNMENT - 1) /* Return the number of bytes in size class I, as a uint. */ #define INDEX2SIZE(I) (((uint)(I) + 1) << ALIGNMENT_SHIFT) @@ -314,14 +313,12 @@ struct arena_object { struct arena_object* prevarena; }; -#undef ROUNDUP -#define ROUNDUP(x) (((x) + ALIGNMENT_MASK) & ~ALIGNMENT_MASK) -#define POOL_OVERHEAD ROUNDUP(sizeof(struct pool_header)) +#define POOL_OVERHEAD _Py_SIZE_ROUND_UP(sizeof(struct pool_header), ALIGNMENT) #define DUMMY_SIZE_IDX 0xffff /* size class of newly cached pools */ /* Round pointer P down to the closest pool-aligned address <= P, as a poolp */ -#define POOL_ADDR(P) ((poolp)((uptr)(P) & ~(uptr)POOL_SIZE_MASK)) +#define POOL_ADDR(P) ((poolp)_Py_ALIGN_DOWN((P), POOL_SIZE)) /* Return total number of blocks in pool of size index I, as a uint. */ #define NUMBLOCKS(I) ((uint)(POOL_SIZE - POOL_OVERHEAD) / INDEX2SIZE(I)) diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index 7d55f49..2a01089 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -2,9 +2,6 @@ #if STRINGLIB_IS_UNICODE -/* Mask to check or force alignment of a pointer to C 'long' boundaries */ -#define LONG_PTR_MASK (size_t) (SIZEOF_LONG - 1) - /* Mask to quickly check whether a C 'long' contains a non-ASCII, UTF8-encoded char. */ #if (SIZEOF_LONG == 8) @@ -25,7 +22,7 @@ STRINGLIB(utf8_decode)(const char **inptr, const char *end, { Py_UCS4 ch; const char *s = *inptr; - const char *aligned_end = (const char *) ((size_t) end & ~LONG_PTR_MASK); + const char *aligned_end = (const char *) _Py_ALIGN_DOWN(end, SIZEOF_LONG); STRINGLIB_CHAR *p = dest + *outpos; while (s < end) { @@ -39,7 +36,7 @@ STRINGLIB(utf8_decode)(const char **inptr, const char *end, First, check if we can do an aligned read, as most CPUs have a penalty for unaligned reads. */ - if (!((size_t) s & LONG_PTR_MASK)) { + if (_Py_IS_ALIGNED(s, SIZEOF_LONG)) { /* Help register allocation */ register const char *_s = s; register STRINGLIB_CHAR *_p = p; @@ -453,7 +450,7 @@ STRINGLIB(utf16_decode)(const unsigned char **inptr, const unsigned char *e, { Py_UCS4 ch; const unsigned char *aligned_end = - (const unsigned char *) ((size_t) e & ~LONG_PTR_MASK); + (const unsigned char *) _Py_ALIGN_DOWN(e, SIZEOF_LONG); const unsigned char *q = *inptr; STRINGLIB_CHAR *p = dest + *outpos; /* Offsets from q for retrieving byte pairs in the right order. */ @@ -468,7 +465,7 @@ STRINGLIB(utf16_decode)(const unsigned char **inptr, const unsigned char *e, Py_UCS4 ch2; /* First check for possible aligned read of a C 'long'. Unaligned reads are more expensive, better to defer to another iteration. */ - if (!((size_t) q & LONG_PTR_MASK)) { + if (_Py_IS_ALIGNED(q, SIZEOF_LONG)) { /* Fast path for runs of in-range non-surrogate chars. */ register const unsigned char *_q = q; while (_q < aligned_end) { @@ -565,7 +562,6 @@ IllegalSurrogate: #undef FAST_CHAR_MASK #undef STRIPPED_MASK #undef SWAB -#undef LONG_PTR_MASK Py_LOCAL_INLINE(void) @@ -588,7 +584,7 @@ STRINGLIB(utf16_encode)(unsigned short *out, _PyUnicode_CONVERT_BYTES(STRINGLIB_CHAR, unsigned short, in, end, out); # endif } else { - const STRINGLIB_CHAR *unrolled_end = in + (len & ~ (Py_ssize_t) 3); + const STRINGLIB_CHAR *unrolled_end = in + _Py_SIZE_ROUND_DOWN(len, 4); while (in < unrolled_end) { out[0] = SWAB2(in[0]); out[1] = SWAB2(in[1]); diff --git a/Objects/stringlib/fastsearch.h b/Objects/stringlib/fastsearch.h index 5b8d5db..ecf885e 100644 --- a/Objects/stringlib/fastsearch.h +++ b/Objects/stringlib/fastsearch.h @@ -43,8 +43,7 @@ STRINGLIB(fastsearch_memchr_1char)(const STRINGLIB_CHAR* s, Py_ssize_t n, #define DO_MEMCHR(memchr, s, needle, nchars) do { \ candidate = memchr((const void *) (s), (needle), (nchars) * sizeof(STRINGLIB_CHAR)); \ - found = (const STRINGLIB_CHAR *) \ - ((Py_ssize_t) candidate & (~ ((Py_ssize_t) sizeof(STRINGLIB_CHAR) - 1))); \ + found = (const STRINGLIB_CHAR *) _Py_ALIGN_DOWN(candidate, sizeof(STRINGLIB_CHAR)); \ } while (0) if (mode == FAST_SEARCH) { diff --git a/Objects/stringlib/find_max_char.h b/Objects/stringlib/find_max_char.h index 9e344a0..06559c8 100644 --- a/Objects/stringlib/find_max_char.h +++ b/Objects/stringlib/find_max_char.h @@ -2,9 +2,6 @@ #if STRINGLIB_IS_UNICODE -/* Mask to check or force alignment of a pointer to C 'long' boundaries */ -#define LONG_PTR_MASK (size_t) (SIZEOF_LONG - 1) - /* Mask to quickly check whether a C 'long' contains a non-ASCII, UTF8-encoded char. */ #if (SIZEOF_LONG == 8) @@ -21,10 +18,11 @@ Py_LOCAL_INLINE(Py_UCS4) STRINGLIB(find_max_char)(const STRINGLIB_CHAR *begin, const STRINGLIB_CHAR *end) { const unsigned char *p = (const unsigned char *) begin; - const unsigned char *aligned_end = (const unsigned char *) ((size_t) end & ~LONG_PTR_MASK); + const unsigned char *aligned_end = + (const unsigned char *) _Py_ALIGN_DOWN(end, SIZEOF_LONG); while (p < end) { - if (!((size_t) p & LONG_PTR_MASK)) { + if (_Py_IS_ALIGNED(p, SIZEOF_LONG)) { /* Help register allocation */ register const unsigned char *_p = p; while (_p < aligned_end) { @@ -43,7 +41,6 @@ STRINGLIB(find_max_char)(const STRINGLIB_CHAR *begin, const STRINGLIB_CHAR *end) return 127; } -#undef LONG_PTR_MASK #undef ASCII_CHAR_MASK #else /* STRINGLIB_SIZEOF_CHAR == 1 */ @@ -72,7 +69,7 @@ STRINGLIB(find_max_char)(const STRINGLIB_CHAR *begin, const STRINGLIB_CHAR *end) register Py_UCS4 mask; Py_ssize_t n = end - begin; const STRINGLIB_CHAR *p = begin; - const STRINGLIB_CHAR *unrolled_end = begin + (n & ~ (Py_ssize_t) 3); + const STRINGLIB_CHAR *unrolled_end = begin + _Py_SIZE_ROUND_DOWN(n, 4); Py_UCS4 max_char; max_char = MAX_CHAR_ASCII; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 61f743e..748508b 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -159,7 +159,7 @@ extern "C" { const from_type *_end = (end); \ Py_ssize_t n = (_end) - (_iter); \ const from_type *_unrolled_end = \ - _iter + (n & ~ (Py_ssize_t) 3); \ + _iter + _Py_SIZE_ROUND_DOWN(n, 4); \ while (_iter < (_unrolled_end)) { \ _to[0] = (to_type) _iter[0]; \ _to[1] = (to_type) _iter[1]; \ @@ -4635,9 +4635,6 @@ PyUnicode_DecodeUTF8(const char *s, #include "stringlib/codecs.h" #include "stringlib/undef.h" -/* Mask to check or force alignment of a pointer to C 'long' boundaries */ -#define LONG_PTR_MASK (size_t) (SIZEOF_LONG - 1) - /* Mask to quickly check whether a C 'long' contains a non-ASCII, UTF8-encoded char. */ #if (SIZEOF_LONG == 8) @@ -4652,11 +4649,11 @@ static Py_ssize_t ascii_decode(const char *start, const char *end, Py_UCS1 *dest) { const char *p = start; - const char *aligned_end = (const char *) ((size_t) end & ~LONG_PTR_MASK); + const char *aligned_end = (const char *) _Py_ALIGN_DOWN(end, SIZEOF_LONG); #if SIZEOF_LONG <= SIZEOF_VOID_P - assert(!((size_t) dest & LONG_PTR_MASK)); - if (!((size_t) p & LONG_PTR_MASK)) { + assert(_Py_IS_ALIGNED(dest, SIZEOF_LONG)); + if (_Py_IS_ALIGNED(p, SIZEOF_LONG)) { /* Fast path, see in STRINGLIB(utf8_decode) for an explanation. */ /* Help register allocation */ @@ -4682,7 +4679,7 @@ ascii_decode(const char *start, const char *end, Py_UCS1 *dest) while (p < end) { /* Fast path, see in STRINGLIB(utf8_decode) in stringlib/codecs.h for an explanation. */ - if (!((size_t) p & LONG_PTR_MASK)) { + if (_Py_IS_ALIGNED(p, SIZEOF_LONG)) { /* Help register allocation */ register const char *_p = p; while (_p < aligned_end) { @@ -5390,7 +5387,7 @@ _PyUnicode_EncodeUTF16(PyObject *str, return NULL; /* output buffer is 2-bytes aligned */ - assert(((Py_uintptr_t)PyBytes_AS_STRING(v) & 1) == 0); + assert(_Py_IS_ALIGNED(PyBytes_AS_STRING(v), 2)); out = (unsigned short *)PyBytes_AS_STRING(v); if (byteorder == 0) *out++ = 0xFEFF; diff --git a/Parser/node.c b/Parser/node.c index 0dea30f..1e4f0da 100644 --- a/Parser/node.c +++ b/Parser/node.c @@ -71,7 +71,7 @@ fancy_roundup(int n) * capacity. The code is tricky to avoid that. */ #define XXXROUNDUP(n) ((n) <= 1 ? (n) : \ - (n) <= 128 ? (((n) + 3) & ~3) : \ + (n) <= 128 ? _Py_SIZE_ROUND_UP((n), 4) : \ fancy_roundup(n)) diff --git a/Python/pyarena.c b/Python/pyarena.c index 5a255ae..bb2fd1e 100644 --- a/Python/pyarena.c +++ b/Python/pyarena.c @@ -12,8 +12,6 @@ #define DEFAULT_BLOCK_SIZE 8192 #define ALIGNMENT 8 -#define ALIGNMENT_MASK (ALIGNMENT - 1) -#define ROUNDUP(x) (((x) + ALIGNMENT_MASK) & ~ALIGNMENT_MASK) typedef struct _block { /* Total number of bytes owned by this block available to pass out. @@ -85,8 +83,8 @@ block_new(size_t size) b->ab_size = size; b->ab_mem = (void *)(b + 1); b->ab_next = NULL; - b->ab_offset = ROUNDUP((Py_uintptr_t)(b->ab_mem)) - - (Py_uintptr_t)(b->ab_mem); + b->ab_offset = (char *)_Py_ALIGN_UP(b->ab_mem, ALIGNMENT) - + (char *)(b->ab_mem); return b; } @@ -104,7 +102,7 @@ block_alloc(block *b, size_t size) { void *p; assert(b); - size = ROUNDUP(size); + size = _Py_SIZE_ROUND_UP(size, ALIGNMENT); if (b->ab_offset + size > b->ab_size) { /* If we need to allocate more memory than will fit in the default block, allocate a one-off block that is -- cgit v0.12