From 1f35ae0a3c7dc2d7709f60e62cb4d0aa7aeae490 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 3 Aug 2013 19:18:38 +0300 Subject: Issue #17998: Fix an internal error in regular expression engine. --- Lib/test/test_re.py | 10 ++++++++++ Misc/NEWS | 2 ++ Modules/_sre.c | 12 ++++++------ Modules/sre.h | 2 +- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 8bc74a2..c84d4ed 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1040,6 +1040,16 @@ class ReTests(unittest.TestCase): with self.assertRaisesRegex(sre_constants.error, '\?foo'): re.compile('(?P)') + def test_issue17998(self): + for reps in '*', '+', '?', '{1}': + for mod in '', '?': + pattern = '.' + reps + mod + 'yz' + self.assertEqual(re.compile(pattern, re.S).findall('xyz'), + ['xyz'], msg=pattern) + pattern = pattern.encode() + self.assertEqual(re.compile(pattern, re.S).findall(b'xyz'), + [b'xyz'], msg=pattern) + def run_re_tests(): from test.re_tests import tests, SUCCEED, FAIL, SYNTAX_ERROR diff --git a/Misc/NEWS b/Misc/NEWS index 27f97b0..00b5a69 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -59,6 +59,8 @@ Core and Builtins Library ------- +- Issue #17998: Fix an internal error in regular expression engine. + - Issue #17557: Fix os.getgroups() to work with the modified behavior of getgroups(2) on OS X 10.8. Original patch by Mateusz Lenik. diff --git a/Modules/_sre.c b/Modules/_sre.c index 19571fb..2ecbcc0 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -997,7 +997,7 @@ entrance: TRACE(("|%p|%p|REPEAT_ONE %d %d\n", ctx->pattern, ctx->ptr, ctx->pattern[1], ctx->pattern[2])); - if (ctx->pattern[1] > (end - ctx->ptr) / state->charsize) + if ((Py_ssize_t) ctx->pattern[1] > (end - ctx->ptr) / state->charsize) RETURN_FAILURE; /* cannot match */ state->ptr = ctx->ptr; @@ -1081,7 +1081,7 @@ entrance: TRACE(("|%p|%p|MIN_REPEAT_ONE %d %d\n", ctx->pattern, ctx->ptr, ctx->pattern[1], ctx->pattern[2])); - if (ctx->pattern[1] > (end - ctx->ptr) / state->charsize) + if ((Py_ssize_t) ctx->pattern[1] > (end - ctx->ptr) / state->charsize) RETURN_FAILURE; /* cannot match */ state->ptr = ctx->ptr; @@ -1180,7 +1180,7 @@ entrance: TRACE(("|%p|%p|MAX_UNTIL %d\n", ctx->pattern, ctx->ptr, ctx->count)); - if (ctx->count < ctx->u.rep->pattern[1]) { + if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) { /* not enough matches */ ctx->u.rep->count = ctx->count; DO_JUMP(JUMP_MAX_UNTIL_1, jump_max_until_1, @@ -1194,7 +1194,7 @@ entrance: RETURN_FAILURE; } - if ((ctx->count < ctx->u.rep->pattern[2] || + if ((ctx->count < (Py_ssize_t) ctx->u.rep->pattern[2] || ctx->u.rep->pattern[2] == SRE_MAXREPEAT) && state->ptr != ctx->u.rep->last_ptr) { /* we may have enough matches, but if we can @@ -1243,7 +1243,7 @@ entrance: TRACE(("|%p|%p|MIN_UNTIL %d %p\n", ctx->pattern, ctx->ptr, ctx->count, ctx->u.rep->pattern)); - if (ctx->count < ctx->u.rep->pattern[1]) { + if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) { /* not enough matches */ ctx->u.rep->count = ctx->count; DO_JUMP(JUMP_MIN_UNTIL_1, jump_min_until_1, @@ -1272,7 +1272,7 @@ entrance: LASTMARK_RESTORE(); - if ((ctx->count >= ctx->u.rep->pattern[2] + if ((ctx->count >= (Py_ssize_t) ctx->u.rep->pattern[2] && ctx->u.rep->pattern[2] != SRE_MAXREPEAT) || state->ptr == ctx->u.rep->last_ptr) RETURN_FAILURE; diff --git a/Modules/sre.h b/Modules/sre.h index 1a6cd56..0a8f0cf 100644 --- a/Modules/sre.h +++ b/Modules/sre.h @@ -19,7 +19,7 @@ #if SIZEOF_SIZE_T > 4 # define SRE_MAXREPEAT (~(SRE_CODE)0) #else -# define SRE_MAXREPEAT ((SRE_CODE)PY_SSIZE_T_MAX + 1u) +# define SRE_MAXREPEAT ((SRE_CODE)PY_SSIZE_T_MAX) #endif typedef struct { -- cgit v0.12 From f6d0aeeadce3f1aea240b900da5e1fbb430257b2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 3 Aug 2013 20:55:06 +0300 Subject: Issue #16741: Fix an error reporting in int(). --- Include/longobject.h | 1 + Lib/test/test_int.py | 47 +++++++++++++++++---------- Misc/NEWS | 2 ++ Objects/abstract.c | 29 +++-------------- Objects/longobject.c | 89 ++++++++++++++++++++++++++++++++++++---------------- 5 files changed, 101 insertions(+), 67 deletions(-) diff --git a/Include/longobject.h b/Include/longobject.h index cd0cf30..bcb93b9 100644 --- a/Include/longobject.h +++ b/Include/longobject.h @@ -84,6 +84,7 @@ PyAPI_FUNC(PyObject *) PyLong_FromString(char *, char **, int); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) PyLong_FromUnicode(Py_UNICODE*, Py_ssize_t, int); PyAPI_FUNC(PyObject *) PyLong_FromUnicodeObject(PyObject *u, int base); +PyAPI_FUNC(PyObject *) _PyLong_FromBytes(const char *, Py_ssize_t, int); #endif #ifndef Py_LIMITED_API diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 703c233..c198bcc 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -73,14 +73,6 @@ class IntTestCases(unittest.TestCase): x = -1-sys.maxsize self.assertEqual(x >> 1, x//2) - self.assertRaises(ValueError, int, '123\0') - self.assertRaises(ValueError, int, '53', 40) - - # SF bug 1545497: embedded NULs were not detected with - # explicit base - self.assertRaises(ValueError, int, '123\0', 10) - self.assertRaises(ValueError, int, '123\x00 245', 20) - x = int('1' * 600) self.assertIsInstance(x, int) @@ -360,14 +352,37 @@ class IntTestCases(unittest.TestCase): int(TruncReturnsBadInt()) def test_error_message(self): - testlist = ('\xbd', '123\xbd', ' 123 456 ') - for s in testlist: - try: - int(s) - except ValueError as e: - self.assertIn(s.strip(), e.args[0]) - else: - self.fail("Expected int(%r) to raise a ValueError", s) + def check(s, base=None): + with self.assertRaises(ValueError, + msg="int(%r, %r)" % (s, base)) as cm: + if base is None: + int(s) + else: + int(s, base) + self.assertEqual(cm.exception.args[0], + "invalid literal for int() with base %d: %r" % + (10 if base is None else base, s)) + + check('\xbd') + check('123\xbd') + check(' 123 456 ') + + check('123\x00') + # SF bug 1545497: embedded NULs were not detected with explicit base + check('123\x00', 10) + check('123\x00 245', 20) + check('123\x00 245', 16) + check('123\x00245', 20) + check('123\x00245', 16) + # byte string with embedded NUL + check(b'123\x00') + check(b'123\x00', 10) + # non-UTF-8 byte string + check(b'123\xbd') + check(b'123\xbd', 10) + # lone surrogate in Unicode string + check('123\ud800') + check('123\ud800', 10) def test_main(): support.run_unittest(IntTestCases) diff --git a/Misc/NEWS b/Misc/NEWS index 00b5a69..10d908c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,8 @@ What's New in Python 3.3.3 release candidate 1? Core and Builtins ----------------- +- Issue #16741: Fix an error reporting in int(). + - Issue #17899: Fix rare file descriptor leak in os.listdir(). - Issue #18552: Check return value of PyArena_AddPyObject() in diff --git a/Objects/abstract.c b/Objects/abstract.c index 7c24724..7f1808f 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -1240,25 +1240,6 @@ convert_integral_to_int(PyObject *integral, const char *error_format) } -/* Add a check for embedded NULL-bytes in the argument. */ -static PyObject * -long_from_string(const char *s, Py_ssize_t len) -{ - char *end; - PyObject *x; - - x = PyLong_FromString((char*)s, &end, 10); - if (x == NULL) - return NULL; - if (end != s + len) { - PyErr_SetString(PyExc_ValueError, - "null byte in argument for int()"); - Py_DECREF(x); - return NULL; - } - return x; -} - PyObject * PyNumber_Long(PyObject *o) { @@ -1306,16 +1287,16 @@ PyNumber_Long(PyObject *o) if (PyBytes_Check(o)) /* need to do extra error checking that PyLong_FromString() - * doesn't do. In particular int('9.5') must raise an - * exception, not truncate the float. + * doesn't do. In particular int('9\x005') must raise an + * exception, not truncate at the null. */ - return long_from_string(PyBytes_AS_STRING(o), - PyBytes_GET_SIZE(o)); + return _PyLong_FromBytes(PyBytes_AS_STRING(o), + PyBytes_GET_SIZE(o), 10); if (PyUnicode_Check(o)) /* The above check is done in PyLong_FromUnicode(). */ return PyLong_FromUnicodeObject(o, 10); if (!PyObject_AsCharBuffer(o, &buffer, &buffer_len)) - return long_from_string(buffer, buffer_len); + return _PyLong_FromBytes(buffer, buffer_len, 10); return type_error("int() argument must be a string or a " "number, not '%.200s'", o); diff --git a/Objects/longobject.c b/Objects/longobject.c index d4dc45a..30ffc94 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -2005,6 +2005,14 @@ long_from_binary_base(char **str, int base) return long_normalize(z); } +/* Parses a long from a bytestring. Leading and trailing whitespace will be + * ignored. + * + * If successful, a PyLong object will be returned and 'pend' will be pointing + * to the first unused byte unless it's NULL. + * + * If unsuccessful, NULL will be returned. + */ PyObject * PyLong_FromString(char *str, char **pend, int base) { @@ -2267,12 +2275,17 @@ digit beyond the first. str++; if (*str != '\0') goto onError; - if (pend) - *pend = str; long_normalize(z); - return (PyObject *) maybe_small_long(z); + z = maybe_small_long(z); + if (z == NULL) + return NULL; + if (pend != NULL) + *pend = str; + return (PyObject *) z; onError: + if (pend != NULL) + *pend = str; Py_XDECREF(z); slen = strlen(orig_str) < 200 ? strlen(orig_str) : 200; strobj = PyUnicode_FromStringAndSize(orig_str, slen); @@ -2285,6 +2298,31 @@ digit beyond the first. return NULL; } +/* Since PyLong_FromString doesn't have a length parameter, + * check here for possible NULs in the string. + * + * Reports an invalid literal as a bytes object. + */ +PyObject * +_PyLong_FromBytes(const char *s, Py_ssize_t len, int base) +{ + PyObject *result, *strobj; + char *end = NULL; + + result = PyLong_FromString((char*)s, &end, base); + if (end == NULL || (result != NULL && end == s + len)) + return result; + Py_XDECREF(result); + strobj = PyBytes_FromStringAndSize(s, Py_MIN(len, 200)); + if (strobj != NULL) { + PyErr_Format(PyExc_ValueError, + "invalid literal for int() with base %d: %R", + base, strobj); + Py_DECREF(strobj); + } + return NULL; +} + PyObject * PyLong_FromUnicode(Py_UNICODE *u, Py_ssize_t length, int base) { @@ -2299,9 +2337,8 @@ PyLong_FromUnicode(Py_UNICODE *u, Py_ssize_t length, int base) PyObject * PyLong_FromUnicodeObject(PyObject *u, int base) { - PyObject *result; - PyObject *asciidig; - char *buffer, *end; + PyObject *result, *asciidig, *strobj; + char *buffer, *end = NULL; Py_ssize_t buflen; asciidig = _PyUnicode_TransformDecimalAndSpaceToASCII(u); @@ -2310,17 +2347,26 @@ PyLong_FromUnicodeObject(PyObject *u, int base) buffer = PyUnicode_AsUTF8AndSize(asciidig, &buflen); if (buffer == NULL) { Py_DECREF(asciidig); - return NULL; + if (!PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) + return NULL; } - result = PyLong_FromString(buffer, &end, base); - if (result != NULL && end != buffer + buflen) { - PyErr_SetString(PyExc_ValueError, - "null byte in argument for int()"); - Py_DECREF(result); - result = NULL; + else { + result = PyLong_FromString(buffer, &end, base); + if (end == NULL || (result != NULL && end == buffer + buflen)) { + Py_DECREF(asciidig); + return result; + } + Py_DECREF(asciidig); + Py_XDECREF(result); } - Py_DECREF(asciidig); - return result; + strobj = PySequence_GetSlice(u, 0, 200); + if (strobj != NULL) { + PyErr_Format(PyExc_ValueError, + "invalid literal for int() with base %d: %R", + base, strobj); + Py_DECREF(strobj); + } + return NULL; } /* forward */ @@ -4308,23 +4354,12 @@ long_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (PyUnicode_Check(x)) return PyLong_FromUnicodeObject(x, (int)base); else if (PyByteArray_Check(x) || PyBytes_Check(x)) { - /* Since PyLong_FromString doesn't have a length parameter, - * check here for possible NULs in the string. */ char *string; - Py_ssize_t size = Py_SIZE(x); if (PyByteArray_Check(x)) string = PyByteArray_AS_STRING(x); else string = PyBytes_AS_STRING(x); - if (strlen(string) != (size_t)size || !size) { - /* We only see this if there's a null byte in x or x is empty, - x is a bytes or buffer, *and* a base is given. */ - PyErr_Format(PyExc_ValueError, - "invalid literal for int() with base %d: %R", - (int)base, x); - return NULL; - } - return PyLong_FromString(string, NULL, (int)base); + return _PyLong_FromBytes(string, Py_SIZE(x), (int)base); } else { PyErr_SetString(PyExc_TypeError, -- cgit v0.12 From 78fa5e38a9cb03181f4bc53d06d41e453fd7077c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sat, 3 Aug 2013 20:09:42 +0200 Subject: Issue #16067: Add description into MSI file to replace installer's temporary name. --- Misc/NEWS | 2 ++ Tools/msi/msi.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index 00b5a69..2450a75 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -304,6 +304,8 @@ Tools/Demos Build ----- +- Issue #16067: Add description into MSI file to replace installer's temporary name. + - Issue #18256: Compilation fix for recent AIX releases. Patch by David Edelsohn. diff --git a/Tools/msi/msi.py b/Tools/msi/msi.py index 2ec6951..5ed025d 100644 --- a/Tools/msi/msi.py +++ b/Tools/msi/msi.py @@ -1410,7 +1410,10 @@ merge(msiname, "SharedCRT", "TARGETDIR", modules) # certname (from config.py) should be (a substring of) # the certificate subject, e.g. "Python Software Foundation" if certname: - os.system('signtool sign /n "%s" /t http://timestamp.verisign.com/scripts/timestamp.dll %s' % (certname, msiname)) + os.system('signtool sign /n "%s" ' + '/t http://timestamp.verisign.com/scripts/timestamp.dll ' + '/d "Python %s" ' + '%s' % (certname, full_current_version, msiname)) if pdbzip: build_pdbzip() -- cgit v0.12