From 2fd3977a9da2675ad5ee693f8c88300e9c6bb0ee Mon Sep 17 00:00:00 2001 From: Bob Ippolito Date: Mon, 29 May 2006 22:55:48 +0000 Subject: struct: modulo math plus warning on all endian-explicit formats for compatibility with older struct usage (ugly) --- Lib/test/test_struct.py | 47 ++++++++++--- Modules/_struct.c | 174 +++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 199 insertions(+), 22 deletions(-) diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 7981a52..701f7fd 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -3,6 +3,7 @@ import test.test_support import struct import array import unittest +import warnings import sys ISBIGENDIAN = sys.byteorder == "big" @@ -10,7 +11,14 @@ del sys verify((struct.pack('=i', 1)[0] == chr(0)) == ISBIGENDIAN, "bigendian determination appears wrong") -PY_STRUCT_RANGE_CHECKING = 1 +try: + import _struct +except ImportError: + PY_STRUCT_RANGE_CHECKING = 0 + PY_STRUCT_WRAPPING = 1 +else: + PY_STRUCT_RANGE_CHECKING = getattr(_struct, '_PY_STRUCT_RANGE_CHECKING', 0) + PY_STRUCT_WRAPPING = getattr(_struct, '_PY_STRUCT_WRAPPING', 0) def string_reverse(s): chars = list(s) @@ -35,12 +43,29 @@ def simple_err(func, *args): def any_err(func, *args): try: func(*args) - except (struct.error, OverflowError, TypeError): + except (struct.error, TypeError): pass else: raise TestFailed, "%s%s did not raise error" % ( func.__name__, args) +def deprecated_err(func, *args): + warnings.filterwarnings("error", r"""^struct.*""", DeprecationWarning) + warnings.filterwarnings("error", r""".*format requires.*""", DeprecationWarning) + try: + try: + func(*args) + except (struct.error, TypeError): + pass + except DeprecationWarning: + if not PY_STRUCT_WRAPPING: + raise TestFailed, "%s%s expected to raise struct.error" % ( + func.__name__, args) + else: + raise TestFailed, "%s%s did not raise error" % ( + func.__name__, args) + finally: + warnings.resetwarnings() simple_err(struct.calcsize, 'Z') @@ -272,8 +297,8 @@ class IntTester: if verbose: print "Skipping buggy range check for code", code else: - any_err(pack, ">" + code, x) - any_err(pack, "<" + code, x) + deprecated_err(pack, ">" + code, x) + deprecated_err(pack, "<" + code, x) # Much the same for unsigned. code = self.unsigned_code @@ -327,8 +352,8 @@ class IntTester: if verbose: print "Skipping buggy range check for code", code else: - any_err(pack, ">" + code, x) - any_err(pack, "<" + code, x) + deprecated_err(pack, ">" + code, x) + deprecated_err(pack, "<" + code, x) def run(self): from random import randrange @@ -448,13 +473,13 @@ def test_1229380(): for endian in ('', '>', '<'): for cls in (int, long): for fmt in ('B', 'H', 'I', 'L'): - any_err(struct.pack, endian + fmt, cls(-1)) + deprecated_err(struct.pack, endian + fmt, cls(-1)) - any_err(struct.pack, endian + 'B', cls(300)) - any_err(struct.pack, endian + 'H', cls(70000)) + deprecated_err(struct.pack, endian + 'B', cls(300)) + deprecated_err(struct.pack, endian + 'H', cls(70000)) - any_err(struct.pack, endian + 'I', sys.maxint * 4L) - any_err(struct.pack, endian + 'L', sys.maxint * 4L) + deprecated_err(struct.pack, endian + 'I', sys.maxint * 4L) + deprecated_err(struct.pack, endian + 'L', sys.maxint * 4L) if PY_STRUCT_RANGE_CHECKING: test_1229380() diff --git a/Modules/_struct.c b/Modules/_struct.c index be641214..163cd5e 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -17,6 +17,20 @@ static PyTypeObject PyStructType; typedef int Py_ssize_t; #endif +/* If PY_STRUCT_WRAPPING is defined, the struct module will wrap all input + numbers for explicit endians such that they fit in the given type, much + like explicit casting in C. A warning will be raised if the number did + not originally fit within the range of the requested type. If it is + not defined, then all range errors and overflow will be struct.error + exceptions. */ + +#define PY_STRUCT_WRAPPING 1 + +#ifdef PY_STRUCT_WRAPPING +static PyObject *pylong_ulong_mask = NULL; +static PyObject *pyint_zero = NULL; +#endif + /* The translation function for each format character is table driven */ typedef struct _formatdef { char format; @@ -195,6 +209,75 @@ get_ulonglong(PyObject *v, unsigned PY_LONG_LONG *p) #endif +#ifdef PY_STRUCT_WRAPPING + +/* Helper routine to get a Python integer and raise the appropriate error + if it isn't one */ + +static int +get_wrapped_long(PyObject *v, long *p) +{ + if (get_long(v, p) < 0) { + if (PyLong_Check(v) && PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyObject *wrapped; + long x; + PyErr_Clear(); + if (PyErr_Warn(PyExc_DeprecationWarning, "struct integer wrapping is deprecated") < 0) + return -1; + wrapped = PyNumber_And(v, pylong_ulong_mask); + if (wrapped == NULL) + return -1; + x = (long)PyLong_AsUnsignedLong(wrapped); + Py_DECREF(wrapped); + if (x == -1 && PyErr_Occurred()) + return -1; + *p = x; + } else { + return -1; + } + } + return 0; +} + +static int +get_wrapped_ulong(PyObject *v, unsigned long *p) +{ + long x = (long)PyLong_AsUnsignedLong(v); + if (x == -1 && PyErr_Occurred()) { + PyObject *wrapped; + PyErr_Clear(); + wrapped = PyNumber_And(v, pylong_ulong_mask); + if (wrapped == NULL) + return -1; + if (PyErr_Warn(PyExc_DeprecationWarning, "struct integer wrapping is deprecated") < 0) { + Py_DECREF(wrapped); + return -1; + } + x = (long)PyLong_AsUnsignedLong(wrapped); + Py_DECREF(wrapped); + if (x == -1 && PyErr_Occurred()) + return -1; + } + *p = (unsigned long)x; + return 0; +} + +#define RANGE_ERROR(x, f, flag, mask) \ + do { \ + if (_range_error(f, flag) < 0) \ + return -1; \ + else \ + (x) &= (mask); \ + } while (0) + +#else + +#define get_wrapped_long get_long +#define get_wrapped_ulong get_ulong +#define RANGE_ERROR(x, f, flag, mask) return _range_error(f, flag) + +#endif + /* Floating point helpers */ static PyObject * @@ -247,6 +330,25 @@ _range_error(const formatdef *f, int is_unsigned) f->format, largest); } +#ifdef PY_STRUCT_WRAPPING + { + PyObject *ptype, *pvalue, *ptraceback; + PyObject *msg; + int rval; + PyErr_Fetch(&ptype, &pvalue, &ptraceback); + assert(pvalue != NULL); + msg = PyObject_Str(pvalue); + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + if (msg == NULL) + return -1; + rval = PyErr_Warn(PyExc_DeprecationWarning, PyString_AS_STRING(msg)); + Py_DECREF(msg); + if (rval == 0) + return 0; + } +#endif return -1; } @@ -707,15 +809,19 @@ bp_int(char *p, PyObject *v, const formatdef *f) { long x; Py_ssize_t i; - if (get_long(v, &x) < 0) + if (get_wrapped_long(v, &x) < 0) return -1; i = f->size; if (i != SIZEOF_LONG) { if ((i == 2) && (x < -32768 || x > 32767)) - return _range_error(f, 0); + RANGE_ERROR(x, f, 0, 0xffffL); #if (SIZEOF_LONG != 4) else if ((i == 4) && (x < -2147483648L || x > 2147483647L)) - return _range_error(f, 0); + RANGE_ERROR(x, f, 0, 0xffffffffL); +#endif +#ifdef PY_STRUCT_WRAPPING + else if ((i == 1) && (x < -128 || x > 127)) + RANGE_ERROR(x, f, 0, 0xffL); #endif } do { @@ -730,14 +836,14 @@ bp_uint(char *p, PyObject *v, const formatdef *f) { unsigned long x; Py_ssize_t i; - if (get_ulong(v, &x) < 0) + if (get_wrapped_ulong(v, &x) < 0) return -1; i = f->size; if (i != SIZEOF_LONG) { unsigned long maxint = 1; maxint <<= (unsigned long)(i * 8); if (x >= maxint) - return _range_error(f, 1); + RANGE_ERROR(x, f, 1, maxint - 1); } do { p[--i] = (char)x; @@ -804,8 +910,14 @@ bp_double(char *p, PyObject *v, const formatdef *f) static formatdef bigendian_table[] = { {'x', 1, 0, NULL}, +#ifdef PY_STRUCT_WRAPPING + /* Native packers do range checking without wrapping. */ + {'b', 1, 0, nu_byte, bp_int}, + {'B', 1, 0, nu_ubyte, bp_uint}, +#else {'b', 1, 0, nu_byte, np_byte}, {'B', 1, 0, nu_ubyte, np_ubyte}, +#endif {'c', 1, 0, nu_char, np_char}, {'s', 1, 0, NULL}, {'p', 1, 0, NULL}, @@ -915,15 +1027,19 @@ lp_int(char *p, PyObject *v, const formatdef *f) { long x; Py_ssize_t i; - if (get_long(v, &x) < 0) + if (get_wrapped_long(v, &x) < 0) return -1; i = f->size; if (i != SIZEOF_LONG) { if ((i == 2) && (x < -32768 || x > 32767)) - return _range_error(f, 0); + RANGE_ERROR(x, f, 0, 0xffffL); #if (SIZEOF_LONG != 4) else if ((i == 4) && (x < -2147483648L || x > 2147483647L)) - return _range_error(f, 0); + RANGE_ERROR(x, f, 0, 0xffffffffL); +#endif +#ifdef PY_STRUCT_WRAPPING + else if ((i == 1) && (x < -128 || x > 127)) + RANGE_ERROR(x, f, 0, 0xffL); #endif } do { @@ -938,14 +1054,14 @@ lp_uint(char *p, PyObject *v, const formatdef *f) { unsigned long x; Py_ssize_t i; - if (get_ulong(v, &x) < 0) + if (get_wrapped_ulong(v, &x) < 0) return -1; i = f->size; if (i != SIZEOF_LONG) { unsigned long maxint = 1; maxint <<= (unsigned long)(i * 8); if (x >= maxint) - return _range_error(f, 1); + RANGE_ERROR(x, f, 1, maxint - 1); } do { *p++ = (char)x; @@ -1012,8 +1128,14 @@ lp_double(char *p, PyObject *v, const formatdef *f) static formatdef lilendian_table[] = { {'x', 1, 0, NULL}, +#ifdef PY_STRUCT_WRAPPING + /* Native packers do range checking without wrapping. */ + {'b', 1, 0, nu_byte, lp_int}, + {'B', 1, 0, nu_ubyte, lp_uint}, +#else {'b', 1, 0, nu_byte, np_byte}, {'B', 1, 0, nu_ubyte, np_ubyte}, +#endif {'c', 1, 0, nu_char, np_char}, {'s', 1, 0, NULL}, {'p', 1, 0, NULL}, @@ -1418,8 +1540,12 @@ s_pack_internal(PyStructObject *soself, PyObject *args, int offset, char* buf) *res = Py_SAFE_DOWNCAST(n, Py_ssize_t, unsigned char); } else { v = PyTuple_GET_ITEM(args, i++); - if (e->pack(res, v, e) < 0) + if (e->pack(res, v, e) < 0) { + if (PyLong_Check(v) && PyErr_ExceptionMatches(PyExc_OverflowError)) + PyErr_SetString(StructError, + "long too large to convert to int"); return -1; + } } } @@ -1614,6 +1740,26 @@ init_struct(void) if (PyType_Ready(&PyStructType) < 0) return; +#ifdef PY_STRUCT_WRAPPING + if (pyint_zero == NULL) { + pyint_zero = PyInt_FromLong(0); + if (pyint_zero == NULL) + return; + } + if (pylong_ulong_mask == NULL) { +#if (SIZEOF_LONG == 4) + pylong_ulong_mask = PyLong_FromString("FFFFFFFF", NULL, 16); +#else + pylong_ulong_mask = PyLong_FromString("FFFFFFFFFFFFFFFF", NULL, 16); +#endif + if (pylong_ulong_mask == NULL) + return; + } + +#else + /* This speed trick can't be used until wrapping goes away, because + native endian always raises exceptions instead of wrapping. */ + /* Check endian and swap in faster functions */ { int one = 1; @@ -1652,6 +1798,7 @@ init_struct(void) native++; } } +#endif /* Add some symbolic constants to the module */ if (StructError == NULL) { @@ -1665,4 +1812,9 @@ init_struct(void) Py_INCREF((PyObject*)&PyStructType); PyModule_AddObject(m, "Struct", (PyObject*)&PyStructType); + + PyModule_AddIntConstant(m, "_PY_STRUCT_RANGE_CHECKING", 1); +#ifdef PY_STRUCT_WRAPPING + PyModule_AddIntConstant(m, "_PY_STRUCT_WRAPPING", 1); +#endif } -- cgit v0.12