From 09e5cf28aef05ad07bf885c1b732eb567470199a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 00:09:18 +0200 Subject: Issue #22117: Use the _PyTime_t API in _datetime.datetime() constructor * Remove _PyTime_gettimeofday() * Add _PyTime_GetSystemClock() --- Include/pytime.h | 20 +++----- Modules/_datetimemodule.c | 18 +++++-- Modules/_testcapimodule.c | 2 +- Python/pytime.c | 119 ++++++---------------------------------------- 4 files changed, 35 insertions(+), 124 deletions(-) diff --git a/Include/pytime.h b/Include/pytime.h index 0ff009a..919ba30 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -13,15 +13,6 @@ functions and constants extern "C" { #endif -#ifdef HAVE_GETTIMEOFDAY -typedef struct timeval _PyTime_timeval; -#else -typedef struct { - time_t tv_sec; /* seconds since Jan. 1, 1970 */ - long tv_usec; /* and microseconds */ -} _PyTime_timeval; -#endif - /* Structure used by time.get_clock_info() */ typedef struct { const char *implementation; @@ -30,11 +21,6 @@ typedef struct { double resolution; } _Py_clock_info_t; -/* Similar to POSIX gettimeofday but cannot fail. If system gettimeofday - * fails or is not available, fall back to lower resolution clocks. - */ -PyAPI_FUNC(void) _PyTime_gettimeofday(_PyTime_timeval *tp); - typedef enum { /* Round towards zero. */ _PyTime_ROUND_DOWN=0, @@ -134,6 +120,12 @@ PyAPI_FUNC(int) _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts); #endif /* Get the current time from the system clock. + + The function cannot fail. _PyTime_Init() ensures that the system clock + works. */ +PyAPI_FUNC(_PyTime_t) _PyTime_GetSystemClock(void); + +/* Get the current time from the system clock. * Fill clock information if info is not NULL. * Raise an exception and return -1 on error, return 0 on success. */ diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 09285d9..c3e54f7 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -7,6 +7,10 @@ #include +#ifdef MS_WINDOWS +# include /* struct timeval */ +#endif + /* Differentiate between building the core module and building extension * modules. */ @@ -4093,6 +4097,8 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, if (_PyTime_ObjectToTimeval(timestamp, &timet, &us, _PyTime_ROUND_DOWN) == -1) return NULL; + assert(0 <= us && us <= 999999); + return datetime_from_timet_and_us(cls, f, timet, (int)us, tzinfo); } @@ -4103,10 +4109,14 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, static PyObject * datetime_best_possible(PyObject *cls, TM_FUNC f, PyObject *tzinfo) { - _PyTime_timeval t; - _PyTime_gettimeofday(&t); - return datetime_from_timet_and_us(cls, f, t.tv_sec, (int)t.tv_usec, - tzinfo); + _PyTime_t ts = _PyTime_GetSystemClock(); + struct timeval tv; + + if (_PyTime_AsTimeval(ts, &tv, _PyTime_ROUND_FLOOR) < 0) + return NULL; + assert(0 <= tv.tv_usec && tv.tv_usec <= 999999); + + return datetime_from_timet_and_us(cls, f, tv.tv_sec, tv.tv_usec, tzinfo); } /*[clinic input] diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 5c54ad6..9abb7cc 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -15,7 +15,7 @@ #include #ifdef MS_WINDOWS -# include +# include /* struct timeval */ #endif #ifdef WITH_THREAD diff --git a/Python/pytime.c b/Python/pytime.c index d23d9d3..11e3a62 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -19,106 +19,6 @@ #define MS_TO_NS (MS_TO_US * US_TO_NS) #define SEC_TO_NS (SEC_TO_MS * MS_TO_NS) -static int -pygettimeofday(_PyTime_timeval *tp, _Py_clock_info_t *info, int raise) -{ -#ifdef MS_WINDOWS - FILETIME system_time; - ULARGE_INTEGER large; - ULONGLONG microseconds; - - assert(info == NULL || raise); - - GetSystemTimeAsFileTime(&system_time); - large.u.LowPart = system_time.dwLowDateTime; - large.u.HighPart = system_time.dwHighDateTime; - /* 11,644,473,600,000,000: number of microseconds between - the 1st january 1601 and the 1st january 1970 (369 years + 89 leap - days). */ - microseconds = large.QuadPart / 10 - 11644473600000000; - tp->tv_sec = microseconds / SEC_TO_US; - tp->tv_usec = microseconds % SEC_TO_US; - if (info) { - DWORD timeAdjustment, timeIncrement; - BOOL isTimeAdjustmentDisabled, ok; - - info->implementation = "GetSystemTimeAsFileTime()"; - info->monotonic = 0; - ok = GetSystemTimeAdjustment(&timeAdjustment, &timeIncrement, - &isTimeAdjustmentDisabled); - if (!ok) { - PyErr_SetFromWindowsErr(0); - return -1; - } - info->resolution = timeIncrement * 1e-7; - info->adjustable = 1; - } - -#else /* MS_WINDOWS */ - int err; -#ifdef HAVE_CLOCK_GETTIME - struct timespec ts; -#endif - - assert(info == NULL || raise); - -#ifdef HAVE_CLOCK_GETTIME - err = clock_gettime(CLOCK_REALTIME, &ts); - if (err) { - if (raise) - PyErr_SetFromErrno(PyExc_OSError); - return -1; - } - tp->tv_sec = ts.tv_sec; - tp->tv_usec = ts.tv_nsec / US_TO_NS; - - if (info) { - struct timespec res; - info->implementation = "clock_gettime(CLOCK_REALTIME)"; - info->monotonic = 0; - info->adjustable = 1; - if (clock_getres(CLOCK_REALTIME, &res) == 0) - info->resolution = res.tv_sec + res.tv_nsec * 1e-9; - else - info->resolution = 1e-9; - } -#else /* HAVE_CLOCK_GETTIME */ - - /* test gettimeofday() */ -#ifdef GETTIMEOFDAY_NO_TZ - err = gettimeofday(tp); -#else - err = gettimeofday(tp, (struct timezone *)NULL); -#endif - if (err) { - if (raise) - PyErr_SetFromErrno(PyExc_OSError); - return -1; - } - - if (info) { - info->implementation = "gettimeofday()"; - info->resolution = 1e-6; - info->monotonic = 0; - info->adjustable = 1; - } -#endif /* !HAVE_CLOCK_GETTIME */ -#endif /* !MS_WINDOWS */ - assert(0 <= tp->tv_usec && tp->tv_usec < SEC_TO_US); - return 0; -} - -void -_PyTime_gettimeofday(_PyTime_timeval *tp) -{ - if (pygettimeofday(tp, NULL, 0) < 0) { - /* cannot happen, _PyTime_Init() checks that pygettimeofday() works */ - assert(0); - tp->tv_sec = 0; - tp->tv_usec = 0; - } -} - static void error_time_t_overflow(void) { @@ -577,6 +477,20 @@ pygettimeofday_new(_PyTime_t *tp, _Py_clock_info_t *info, int raise) return 0; } +_PyTime_t +_PyTime_GetSystemClock(void) +{ + _PyTime_t t; + if (pygettimeofday_new(&t, NULL, 0) < 0) { + /* should not happen, _PyTime_Init() checked the clock at startup */ + assert(0); + + /* use a fixed value instead of a random value from the stack */ + t = 0; + } + return t; +} + int _PyTime_GetSystemClockWithInfo(_PyTime_t *t, _Py_clock_info_t *info) { @@ -715,14 +629,9 @@ _PyTime_GetMonotonicClockWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) int _PyTime_Init(void) { - _PyTime_timeval tv; _PyTime_t t; /* ensure that the system clock works */ - if (pygettimeofday(&tv, NULL, 1) < 0) - return -1; - - /* ensure that the system clock works */ if (_PyTime_GetSystemClockWithInfo(&t, NULL) < 0) return -1; -- cgit v0.12 From 1bd18ba9a78c58b817564637f1937c2bc3920ecd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 00:25:38 +0200 Subject: Issue #22117: Cleanup pytime.c/.h --- Include/pytime.h | 74 ++++++++++++++++++++++++++++---------------------------- Python/pytime.c | 12 ++++----- 2 files changed, 42 insertions(+), 44 deletions(-) diff --git a/Include/pytime.h b/Include/pytime.h index 919ba30..b872774 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -13,13 +13,16 @@ functions and constants extern "C" { #endif -/* Structure used by time.get_clock_info() */ -typedef struct { - const char *implementation; - int monotonic; - int adjustable; - double resolution; -} _Py_clock_info_t; +#ifdef PY_INT64_T +/* _PyTime_t: Python timestamp with subsecond precision. It can be used to + store a duration, and so indirectly a date (related to another date, like + UNIX epoch). */ +typedef PY_INT64_T _PyTime_t; +#define _PyTime_MIN PY_LLONG_MIN +#define _PyTime_MAX PY_LLONG_MAX +#else +# error "_PyTime_t need signed 64-bit integer type" +#endif typedef enum { /* Round towards zero. */ @@ -32,12 +35,6 @@ typedef enum { _PyTime_ROUND_FLOOR } _PyTime_round_t; -/* Convert a number of seconds, int or float, to time_t. */ -PyAPI_FUNC(int) _PyTime_ObjectToTime_t( - PyObject *obj, - time_t *sec, - _PyTime_round_t); - /* Convert a time_t to a PyLong. */ PyAPI_FUNC(PyObject *) _PyLong_FromTime_t( time_t sec); @@ -46,6 +43,12 @@ PyAPI_FUNC(PyObject *) _PyLong_FromTime_t( PyAPI_FUNC(time_t) _PyLong_AsTime_t( PyObject *obj); +/* Convert a number of seconds, int or float, to time_t. */ +PyAPI_FUNC(int) _PyTime_ObjectToTime_t( + PyObject *obj, + time_t *sec, + _PyTime_round_t); + /* Convert a number of seconds, int or float, to a timeval structure. usec is in the range [0; 999999] and rounded towards zero. For example, -1.2 is converted to (-2, 800000). */ @@ -64,22 +67,6 @@ PyAPI_FUNC(int) _PyTime_ObjectToTimespec( long *nsec, _PyTime_round_t); -/* Initialize time. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) _PyTime_Init(void); - -/****************** NEW _PyTime_t API **********************/ - -#ifdef PY_INT64_T -/* _PyTime_t: Python timestamp with subsecond precision. It can be used to - store a duration, and so indirectly a date (related to another date, like - UNIX epoch). */ -typedef PY_INT64_T _PyTime_t; -#define _PyTime_MIN PY_LLONG_MIN -#define _PyTime_MAX PY_LLONG_MAX -#else -# error "_PyTime_t need signed 64-bit integer type" -#endif /* Create a timestamp from a number of nanoseconds (C long). */ PyAPI_FUNC(_PyTime_t) _PyTime_FromNanoseconds(PY_LONG_LONG ns); @@ -125,14 +112,6 @@ PyAPI_FUNC(int) _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts); works. */ PyAPI_FUNC(_PyTime_t) _PyTime_GetSystemClock(void); -/* Get the current time from the system clock. - * Fill clock information if info is not NULL. - * Raise an exception and return -1 on error, return 0 on success. - */ -PyAPI_FUNC(int) _PyTime_GetSystemClockWithInfo( - _PyTime_t *t, - _Py_clock_info_t *info); - /* Get the time of a monotonic clock, i.e. a clock that cannot go backwards. The clock is not affected by system clock updates. The reference point of the returned value is undefined, so that only the difference between the @@ -142,6 +121,23 @@ PyAPI_FUNC(int) _PyTime_GetSystemClockWithInfo( is available and works. */ PyAPI_FUNC(_PyTime_t) _PyTime_GetMonotonicClock(void); + +/* Structure used by time.get_clock_info() */ +typedef struct { + const char *implementation; + int monotonic; + int adjustable; + double resolution; +} _Py_clock_info_t; + +/* Get the current time from the system clock. + * Fill clock information if info is not NULL. + * Raise an exception and return -1 on error, return 0 on success. + */ +PyAPI_FUNC(int) _PyTime_GetSystemClockWithInfo( + _PyTime_t *t, + _Py_clock_info_t *info); + /* Get the time of a monotonic clock, i.e. a clock that cannot go backwards. The clock is not affected by system clock updates. The reference point of the returned value is undefined, so that only the difference between the @@ -155,6 +151,10 @@ PyAPI_FUNC(int) _PyTime_GetMonotonicClockWithInfo( _Py_clock_info_t *info); +/* Initialize time. + Return 0 on success, raise an exception and return -1 on error. */ +PyAPI_FUNC(int) _PyTime_Init(void); + #ifdef __cplusplus } #endif diff --git a/Python/pytime.c b/Python/pytime.c index 11e3a62..d9ff3c6 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -151,8 +151,6 @@ _PyTime_ObjectToTimeval(PyObject *obj, time_t *sec, long *usec, return _PyTime_ObjectToDenominator(obj, sec, usec, 1e6, round); } -/****************** NEW _PyTime_t API **********************/ - static void _PyTime_overflow(void) { @@ -161,7 +159,7 @@ _PyTime_overflow(void) } int -_PyTime_RoundTowardsInfinity(int is_neg, _PyTime_round_t round) +_PyTime_RoundTowardsPosInf(int is_neg, _PyTime_round_t round) { if (round == _PyTime_ROUND_FLOOR) return 0; @@ -196,7 +194,7 @@ _PyTime_FromTimespec(_PyTime_t *tp, struct timespec *ts, int raise) *tp = t; return res; } -#else +#elif !defined(MS_WINDOWS) static int _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv, int raise) { @@ -227,7 +225,7 @@ _PyTime_FromSecondsObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round) d = PyFloat_AsDouble(obj); d *= 1e9; - if (_PyTime_RoundTowardsInfinity(d < 0, round)) + if (_PyTime_RoundTowardsPosInf(d < 0, round)) d = ceil(d); else d = floor(d); @@ -293,7 +291,7 @@ _PyTime_Multiply(_PyTime_t t, unsigned int multiply, _PyTime_round_t round) _PyTime_t k; if (multiply < SEC_TO_NS) { k = SEC_TO_NS / multiply; - if (_PyTime_RoundTowardsInfinity(t < 0, round)) + if (_PyTime_RoundTowardsPosInf(t < 0, round)) return (t + k - 1) / k; else return t / k; @@ -353,7 +351,7 @@ _PyTime_AsTimeval(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) res = -1; #endif - if (_PyTime_RoundTowardsInfinity(tv->tv_sec < 0, round)) + if (_PyTime_RoundTowardsPosInf(tv->tv_sec < 0, round)) tv->tv_usec = (int)((ns + US_TO_NS - 1) / US_TO_NS); else tv->tv_usec = (int)(ns / US_TO_NS); -- cgit v0.12 From a3a100b594982f10911f2c9db6cf954bb8ff8f20 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 30 Mar 2015 01:28:02 +0300 Subject: Issue #22390: test.regrtest now emits a warning if temporary files or directories are left after running a test. --- Lib/test/regrtest.py | 26 +++++++++++--------------- Misc/NEWS | 3 +++ 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index b633631..350e684 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -1031,7 +1031,7 @@ class saved_test_environment: # to a thread, so check processes first. 'multiprocessing.process._dangling', 'threading._dangling', 'sysconfig._CONFIG_VARS', 'sysconfig._INSTALL_SCHEMES', - 'support.TESTFN', 'locale', 'warnings.showwarning', + 'files', 'locale', 'warnings.showwarning', ) def get_sys_argv(self): @@ -1187,20 +1187,16 @@ class saved_test_environment: sysconfig._INSTALL_SCHEMES.clear() sysconfig._INSTALL_SCHEMES.update(saved[2]) - def get_support_TESTFN(self): - if os.path.isfile(support.TESTFN): - result = 'f' - elif os.path.isdir(support.TESTFN): - result = 'd' - else: - result = None - return result - def restore_support_TESTFN(self, saved_value): - if saved_value is None: - if os.path.isfile(support.TESTFN): - os.unlink(support.TESTFN) - elif os.path.isdir(support.TESTFN): - shutil.rmtree(support.TESTFN) + def get_files(self): + return sorted(fn + ('/' if os.path.isdir(fn) else '') + for fn in os.listdir()) + def restore_files(self, saved_value): + fn = support.TESTFN + if fn not in saved_value and (fn + '/') not in saved_value: + if os.path.isfile(fn): + support.unlink(fn) + elif os.path.isdir(fn): + support.rmtree(fn) _lc = [getattr(locale, lc) for lc in dir(locale) if lc.startswith('LC_')] diff --git a/Misc/NEWS b/Misc/NEWS index 48d1801..b80cbe2 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -130,6 +130,9 @@ Library Tests ----- +- Issue #22390: test.regrtest now emits a warning if temporary files or + directories are left after running a test. + - Issue #23583: Added tests for standard IO streams in IDLE. Build -- cgit v0.12 From f81f0f9c63c8ae306d550b5bb387abf30e60a668 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 00:44:06 +0200 Subject: Issue #22117: Fix rounding and implement _PyTime_ROUND_FLOOR in: - _PyTime_ObjectToTime_t() - _PyTime_ObjectToTimespec() - _PyTime_ObjectToTimeval() --- Lib/test/test_time.py | 4 ++-- Python/pytime.c | 44 ++++++++++++++++++++------------------------ 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index b0c97d5..bcbf41d 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -647,13 +647,13 @@ class TestPytime(unittest.TestCase): (1e-9, (0, 1), _PyTime.ROUND_DOWN), (1e-10, (0, 0), _PyTime.ROUND_DOWN), (-1e-9, (-1, 999999999), _PyTime.ROUND_DOWN), - (-1e-10, (-1, 999999999), _PyTime.ROUND_DOWN), + (-1e-10, (0, 0), _PyTime.ROUND_DOWN), (-1.2, (-2, 800000000), _PyTime.ROUND_DOWN), (0.9999999999, (0, 999999999), _PyTime.ROUND_DOWN), (1.1234567890, (1, 123456789), _PyTime.ROUND_DOWN), (1.1234567899, (1, 123456789), _PyTime.ROUND_DOWN), (-1.1234567890, (-2, 876543211), _PyTime.ROUND_DOWN), - (-1.1234567891, (-2, 876543210), _PyTime.ROUND_DOWN), + (-1.1234567891, (-2, 876543211), _PyTime.ROUND_DOWN), # Round away from zero (0, (0, 0), _PyTime.ROUND_UP), (-1, (-1, 0), _PyTime.ROUND_UP), diff --git a/Python/pytime.c b/Python/pytime.c index d9ff3c6..2bf6ba5 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -26,6 +26,14 @@ error_time_t_overflow(void) "timestamp out of range for platform time_t"); } +static int +_PyTime_RoundTowardsPosInf(int is_neg, _PyTime_round_t round) +{ + if (round == _PyTime_ROUND_FLOOR) + return 0; + return ((round == _PyTime_ROUND_UP) ^ is_neg); +} + time_t _PyLong_AsTime_t(PyObject *obj) { @@ -74,18 +82,16 @@ _PyTime_ObjectToDenominator(PyObject *obj, time_t *sec, long *numerator, } floatpart *= denominator; - if (round == _PyTime_ROUND_UP) { - if (intpart >= 0) { - floatpart = ceil(floatpart); - if (floatpart >= denominator) { - floatpart = 0.0; - intpart += 1.0; - } - } - else { - floatpart = floor(floatpart); + if (_PyTime_RoundTowardsPosInf(intpart < 0, round)) { + floatpart = ceil(floatpart); + if (floatpart >= denominator) { + floatpart = 0.0; + intpart += 1.0; } } + else { + floatpart = floor(floatpart); + } *sec = (time_t)intpart; err = intpart - (double)*sec; @@ -113,12 +119,10 @@ _PyTime_ObjectToTime_t(PyObject *obj, time_t *sec, _PyTime_round_t round) double d, intpart, err; d = PyFloat_AsDouble(obj); - if (round == _PyTime_ROUND_UP) { - if (d >= 0) - d = ceil(d); - else - d = floor(d); - } + if (_PyTime_RoundTowardsPosInf(d < 0, round)) + d = ceil(d); + else + d = floor(d); (void)modf(d, &intpart); *sec = (time_t)intpart; @@ -158,14 +162,6 @@ _PyTime_overflow(void) "timestamp too large to convert to C _PyTime_t"); } -int -_PyTime_RoundTowardsPosInf(int is_neg, _PyTime_round_t round) -{ - if (round == _PyTime_ROUND_FLOOR) - return 0; - return ((round == _PyTime_ROUND_UP) ^ is_neg); -} - _PyTime_t _PyTime_FromNanoseconds(PY_LONG_LONG ns) { -- cgit v0.12 From dca028b86ade11441554f8cdb9d2ae56c119b413 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 01:02:57 +0200 Subject: Issue #22117: Fix os.utime(), it now rounds the timestamp towards minus infinity (-inf) instead of rounding towards zero. Replace _PyTime_ROUND_DOWN with _PyTime_ROUND_FLOOR. --- Misc/NEWS | 3 +++ Modules/posixmodule.c | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 96621a1..e96d66e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -30,6 +30,9 @@ Core and Builtins Library ------- +- Issue #22117: Fix os.utime(), it now rounds the timestamp towards minus + infinity (-inf) instead of rounding towards zero. + - Issue #14260: The groupindex attribute of regular expression pattern object now is non-modifiable mapping. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 9a44d46..801305f 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -6127,9 +6127,9 @@ os_utime_impl(PyModuleDef *module, path_t *path, PyObject *times, PyObject *ns, } utime.now = 0; if (_PyTime_ObjectToTimespec(PyTuple_GET_ITEM(times, 0), - &a_sec, &a_nsec, _PyTime_ROUND_DOWN) == -1 || + &a_sec, &a_nsec, _PyTime_ROUND_FLOOR) == -1 || _PyTime_ObjectToTimespec(PyTuple_GET_ITEM(times, 1), - &m_sec, &m_nsec, _PyTime_ROUND_DOWN) == -1) { + &m_sec, &m_nsec, _PyTime_ROUND_FLOOR) == -1) { goto exit; } utime.atime_s = a_sec; -- cgit v0.12 From e4a994d6171f47ee9ba68ae1484d940349d62564 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 01:10:14 +0200 Subject: Issue #22117: Fix rounding of fromtimestamp() methods of datetime.datetime and datetime.time: round towards minus infinity ("floor") instead of rounding towards zero ("down"). --- Modules/_datetimemodule.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index c3e54f7..ab2acae 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -2463,7 +2463,7 @@ date_local_from_object(PyObject *cls, PyObject *obj) struct tm *tm; time_t t; - if (_PyTime_ObjectToTime_t(obj, &t, _PyTime_ROUND_DOWN) == -1) + if (_PyTime_ObjectToTime_t(obj, &t, _PyTime_ROUND_FLOOR) == -1) return NULL; tm = localtime(&t); @@ -4095,7 +4095,8 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, time_t timet; long us; - if (_PyTime_ObjectToTimeval(timestamp, &timet, &us, _PyTime_ROUND_DOWN) == -1) + if (_PyTime_ObjectToTimeval(timestamp, + &timet, &us, _PyTime_ROUND_FLOOR) == -1) return NULL; assert(0 <= us && us <= 999999); -- cgit v0.12 From 160e819a1d0a01fe79b66bf398c925c0dac0ded1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 02:18:31 +0200 Subject: Issue #23694: Fix usage of _Py_open() in the _posixsubprocess module Don't call _Py_open() from _close_open_fds_safe() because it is call just after fork(). It's not good to play with locks (the GIL) between fork() and exec(). Use instead _Py_open_noraise() which doesn't touch to the GIL. --- Modules/_posixsubprocess.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index a33df21..0b385a1 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -254,10 +254,9 @@ _close_open_fds_safe(int start_fd, PyObject* py_fds_to_keep) { int fd_dir_fd; - fd_dir_fd = _Py_open(FD_DIR, O_RDONLY); + fd_dir_fd = _Py_open_noraise(FD_DIR, O_RDONLY); if (fd_dir_fd == -1) { /* No way to get a list of open fds. */ - PyErr_Clear(); _close_fds_by_brute_force(start_fd, py_fds_to_keep); return; } else { -- cgit v0.12 From ea9c0dd2c27884691f0a0af983fd41d4d818e93f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 02:51:13 +0200 Subject: Issue #22117: Fix usage of _PyTime_AsTimeval() Add _PyTime_AsTimeval_noraise() function. Call it when it's not possible (or not useful) to raise a Python exception on overflow. --- Include/pytime.h | 8 +++++++- Modules/_ssl.c | 4 +--- Modules/_testcapimodule.c | 5 +---- Modules/socketmodule.c | 8 ++------ Modules/timemodule.c | 5 +---- Python/pytime.c | 19 +++++++++++++++++-- 6 files changed, 29 insertions(+), 20 deletions(-) diff --git a/Include/pytime.h b/Include/pytime.h index b872774..f14e1fc 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -94,11 +94,17 @@ PyAPI_FUNC(PyObject *) _PyTime_AsNanosecondsObject(_PyTime_t t); /* Convert a timestamp to a timeval structure (microsecond resolution). tv_usec is always positive. - Return -1 if the conversion overflowed, return 0 on success. */ + Raise an exception and return -1 if the conversion overflowed, + return 0 on success. */ PyAPI_FUNC(int) _PyTime_AsTimeval(_PyTime_t t, struct timeval *tv, _PyTime_round_t round); +/* Similar to _PyTime_AsTimeval(), but don't raise an exception on error. */ +PyAPI_FUNC(int) _PyTime_AsTimeval_noraise(_PyTime_t t, + struct timeval *tv, + _PyTime_round_t round); + #if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) /* Convert a timestamp to a timespec structure (nanosecond resolution). tv_nsec is always positive. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 54f5d14..217c77c 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -1651,9 +1651,7 @@ check_socket_and_wait_for_timeout(PySocketSockObject *s, int writing) if (!_PyIsSelectable_fd(s->sock_fd)) return SOCKET_TOO_LARGE_FOR_SELECT; - /* conversion was already checked for overflow when - the timeout was set */ - (void)_PyTime_AsTimeval(s->sock_timeout, &tv, _PyTime_ROUND_UP); + _PyTime_AsTimeval_noraise(s->sock_timeout, &tv, _PyTime_ROUND_UP); FD_ZERO(&fds); FD_SET(s->sock_fd, &fds); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 9abb7cc..128ba09 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3427,11 +3427,8 @@ test_PyTime_AsTimeval(PyObject *self, PyObject *args) if (check_time_rounding(round) < 0) return NULL; t = _PyTime_FromNanoseconds(ns); - if (_PyTime_AsTimeval(t, &tv, round) < 0) { - PyErr_SetString(PyExc_OverflowError, - "timeout doesn't fit into C timeval"); + if (_PyTime_AsTimeval(t, &tv, round) < 0) return NULL; - } seconds = PyLong_FromLong((PY_LONG_LONG)tv.tv_sec); if (seconds == NULL) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 93dcd41..513405e 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -641,9 +641,7 @@ internal_select_ex(PySocketSockObject *s, int writing, _PyTime_t interval) n = poll(&pollfd, 1, timeout_int); Py_END_ALLOW_THREADS; #else - /* conversion was already checked for overflow when - the timeout was set */ - (void)_PyTime_AsTimeval(interval, &tv, _PyTime_ROUND_UP); + _PyTime_AsTimeval_noraise(interval, &tv, _PyTime_ROUND_UP); FD_ZERO(&fds); FD_SET(s->sock_fd, &fds); @@ -2454,9 +2452,7 @@ internal_connect(PySocketSockObject *s, struct sockaddr *addr, int addrlen, struct timeval tv; int conv; - /* conversion was already checked for overflow when - the timeout was set */ - (void)_PyTime_AsTimeval(s->sock_timeout, &tv, _PyTime_ROUND_UP); + _PyTime_AsTimeval_noraise(s->sock_timeout, &tv, _PyTime_ROUND_UP); Py_BEGIN_ALLOW_THREADS FD_ZERO(&fds); diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 99e83cc..3ed3fb3 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -1405,11 +1405,8 @@ pysleep(_PyTime_t secs) do { #ifndef MS_WINDOWS - if (_PyTime_AsTimeval(secs, &timeout, _PyTime_ROUND_UP) < 0) { - PyErr_SetString(PyExc_OverflowError, - "delay doesn't fit into C timeval"); + if (_PyTime_AsTimeval(secs, &timeout, _PyTime_ROUND_UP) < 0) return -1; - } Py_BEGIN_ALLOW_THREADS err = select(0, (fd_set *)0, (fd_set *)0, (fd_set *)0, &timeout); diff --git a/Python/pytime.c b/Python/pytime.c index 2bf6ba5..a7eda86 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -311,8 +311,9 @@ _PyTime_AsMicroseconds(_PyTime_t t, _PyTime_round_t round) return _PyTime_Multiply(t, 1000 * 1000, round); } -int -_PyTime_AsTimeval(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) +static int +_PyTime_AsTimeval_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, + int raise) { _PyTime_t secs, ns; int res = 0; @@ -357,9 +358,23 @@ _PyTime_AsTimeval(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) tv->tv_sec += 1; } + if (res && raise) + _PyTime_overflow(); return res; } +int +_PyTime_AsTimeval(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) +{ + return _PyTime_AsTimeval_impl(t, tv, round, 1); +} + +int +_PyTime_AsTimeval_noraise(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) +{ + return _PyTime_AsTimeval_impl(t, tv, round, 0); +} + #if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) int _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts) -- cgit v0.12 From edddf991d9ea88d16c8ebef7a5f829822d8025fa Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 02:54:57 +0200 Subject: Issue #22117: Add assertions to _PyTime_AsTimeval() and _PyTime_AsTimespec() to check that microseconds and nanoseconds fits into the specified range. --- Python/pytime.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Python/pytime.c b/Python/pytime.c index a7eda86..0d28911 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -360,6 +360,8 @@ _PyTime_AsTimeval_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, if (res && raise) _PyTime_overflow(); + + assert(0 <= tv->tv_usec && tv->tv_usec <= 999999); return res; } @@ -393,6 +395,8 @@ _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts) return -1; } ts->tv_nsec = nsec; + + assert(0 <= ts->tv_nsec && ts->tv_nsec <= 999999999); return 0; } #endif -- cgit v0.12 From 520bddf79a06c06d7b293408a6e3cc119f808afb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 03:21:06 +0200 Subject: Issue #23752: When built from an existing file descriptor, io.FileIO() now only calls fstat() once. Before fstat() was called twice, which was not necessary. --- Misc/NEWS | 26 ++++++++++++++++++++++++++ Modules/_io/fileio.c | 24 ------------------------ 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index e96d66e..2b9ce2e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,6 +2,32 @@ Python News +++++++++++ +What's New in Python 3.5.0 alpha 4? +=================================== + +Release date: XXX + +Core and Builtins +----------------- + +Library +------- + +- Issue #23752: When built from an existing file descriptor, io.FileIO() now + only calls fstat() once. Before fstat() was called twice, which was not + necessary. + +Build +----- + +Tests +----- + +Tools/Demos +----------- + + + What's New in Python 3.5.0 alpha 3? =================================== diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index b35a51b..595f99e 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -177,28 +177,6 @@ fileio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *) self; } -static int -check_fd(int fd) -{ - struct _Py_stat_struct buf; - if (_Py_fstat(fd, &buf) < 0 && -#ifdef MS_WINDOWS - GetLastError() == ERROR_INVALID_HANDLE -#else - errno == EBADF -#endif - ) { - PyObject *exc; - char *msg = strerror(EBADF); - exc = PyObject_CallFunction(PyExc_OSError, "(is)", - EBADF, msg); - PyErr_SetObject(PyExc_OSError, exc); - Py_XDECREF(exc); - return -1; - } - return 0; -} - #ifdef O_CLOEXEC extern int _Py_open_cloexec_works; #endif @@ -355,8 +333,6 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) #endif if (fd >= 0) { - if (check_fd(fd)) - goto error; self->fd = fd; self->closefd = closefd; } -- cgit v0.12 From bcdd777d3c01a6db3b4357922663624ef617e65a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 03:52:49 +0200 Subject: Issue #22117: Add _PyTime_ROUND_CEILING rounding method for timestamps Add also more tests for ROUNd_FLOOR. --- Include/pytime.h | 5 +++- Lib/test/test_time.py | 65 ++++++++++++++++++++++++++++++++++++++++++++--- Modules/_testcapimodule.c | 2 +- Python/pytime.c | 2 ++ 4 files changed, 68 insertions(+), 6 deletions(-) diff --git a/Include/pytime.h b/Include/pytime.h index f14e1fc..ff3e203 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -32,7 +32,10 @@ typedef enum { _PyTime_ROUND_UP, /* Round towards minus infinity (-inf). For example, used to read a clock. */ - _PyTime_ROUND_FLOOR + _PyTime_ROUND_FLOOR, + /* Round towards infinity (+inf). + For example, used for timeout to wait "at least" N seconds. */ + _PyTime_ROUND_CEILING } _PyTime_round_t; /* Convert a time_t to a PyLong. */ diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index bcbf41d..472110c 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -28,13 +28,16 @@ class _PyTime(enum.IntEnum): ROUND_DOWN = 0 # Round away from zero ROUND_UP = 1 - # Round towards -Infinity + # Round towards minus infinity (-inf) ROUND_FLOOR = 2 + # Round towards infinity (+inf) + ROUND_CEILING = 3 ALL_ROUNDING_METHODS = ( _PyTime.ROUND_UP, _PyTime.ROUND_DOWN, - _PyTime.ROUND_FLOOR) + _PyTime.ROUND_FLOOR, + _PyTime.ROUND_CEILING) class TimeTestCase(unittest.TestCase): @@ -621,6 +624,13 @@ class TestPytime(unittest.TestCase): (-1.9, -1, _PyTime.ROUND_DOWN), (1.0, 1, _PyTime.ROUND_DOWN), (1.9, 1, _PyTime.ROUND_DOWN), + # Round towards minus infinity (-inf) + (0, 0, _PyTime.ROUND_FLOOR), + (-1, -1, _PyTime.ROUND_FLOOR), + (-1.0, -1, _PyTime.ROUND_FLOOR), + (-1.9, -2, _PyTime.ROUND_FLOOR), + (1.0, 1, _PyTime.ROUND_FLOOR), + (1.9, 1, _PyTime.ROUND_FLOOR), # Round away from zero (0, 0, _PyTime.ROUND_UP), (-1, -1, _PyTime.ROUND_UP), @@ -628,10 +638,17 @@ class TestPytime(unittest.TestCase): (-1.9, -2, _PyTime.ROUND_UP), (1.0, 1, _PyTime.ROUND_UP), (1.9, 2, _PyTime.ROUND_UP), + # Round towards infinity (+inf) + (0, 0, _PyTime.ROUND_CEILING), + (-1, -1, _PyTime.ROUND_CEILING), + (-1.0, -1, _PyTime.ROUND_CEILING), + (-1.9, -1, _PyTime.ROUND_CEILING), + (1.0, 1, _PyTime.ROUND_CEILING), + (1.9, 2, _PyTime.ROUND_CEILING), ): self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) - rnd = _PyTime.ROUND_DOWN + rnd = _PyTime.ROUND_FLOOR for invalid in self.invalid_values: self.assertRaises(OverflowError, pytime_object_to_time_t, invalid, rnd) @@ -654,6 +671,20 @@ class TestPytime(unittest.TestCase): (1.1234567899, (1, 123456789), _PyTime.ROUND_DOWN), (-1.1234567890, (-2, 876543211), _PyTime.ROUND_DOWN), (-1.1234567891, (-2, 876543211), _PyTime.ROUND_DOWN), + # Round towards minus infinity (-inf) + (0, (0, 0), _PyTime.ROUND_FLOOR), + (-1, (-1, 0), _PyTime.ROUND_FLOOR), + (-1.0, (-1, 0), _PyTime.ROUND_FLOOR), + (1e-9, (0, 1), _PyTime.ROUND_FLOOR), + (1e-10, (0, 0), _PyTime.ROUND_FLOOR), + (-1e-9, (-1, 999999999), _PyTime.ROUND_FLOOR), + (-1e-10, (-1, 999999999), _PyTime.ROUND_FLOOR), + (-1.2, (-2, 800000000), _PyTime.ROUND_FLOOR), + (0.9999999999, (0, 999999999), _PyTime.ROUND_FLOOR), + (1.1234567890, (1, 123456789), _PyTime.ROUND_FLOOR), + (1.1234567899, (1, 123456789), _PyTime.ROUND_FLOOR), + (-1.1234567890, (-2, 876543211), _PyTime.ROUND_FLOOR), + (-1.1234567891, (-2, 876543210), _PyTime.ROUND_FLOOR), # Round away from zero (0, (0, 0), _PyTime.ROUND_UP), (-1, (-1, 0), _PyTime.ROUND_UP), @@ -668,11 +699,25 @@ class TestPytime(unittest.TestCase): (1.1234567899, (1, 123456790), _PyTime.ROUND_UP), (-1.1234567890, (-2, 876543211), _PyTime.ROUND_UP), (-1.1234567891, (-2, 876543210), _PyTime.ROUND_UP), + # Round towards infinity (+inf) + (0, (0, 0), _PyTime.ROUND_CEILING), + (-1, (-1, 0), _PyTime.ROUND_CEILING), + (-1.0, (-1, 0), _PyTime.ROUND_CEILING), + (1e-9, (0, 1), _PyTime.ROUND_CEILING), + (1e-10, (0, 1), _PyTime.ROUND_CEILING), + (-1e-9, (-1, 999999999), _PyTime.ROUND_CEILING), + (-1e-10, (0, 0), _PyTime.ROUND_CEILING), + (-1.2, (-2, 800000000), _PyTime.ROUND_CEILING), + (0.9999999999, (1, 0), _PyTime.ROUND_CEILING), + (1.1234567890, (1, 123456790), _PyTime.ROUND_CEILING), + (1.1234567899, (1, 123456790), _PyTime.ROUND_CEILING), + (-1.1234567890, (-2, 876543211), _PyTime.ROUND_CEILING), + (-1.1234567891, (-2, 876543211), _PyTime.ROUND_CEILING), ): with self.subTest(obj=obj, round=rnd, timespec=timespec): self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) - rnd = _PyTime.ROUND_DOWN + rnd = _PyTime.ROUND_FLOOR for invalid in self.invalid_values: self.assertRaises(OverflowError, pytime_object_to_timespec, invalid, rnd) @@ -794,27 +839,34 @@ class TestPyTime_t(unittest.TestCase): UP = _PyTime.ROUND_UP DOWN = _PyTime.ROUND_DOWN FLOOR = _PyTime.ROUND_FLOOR + CEILING = _PyTime.ROUND_CEILING for obj, ts, rnd in ( # close to zero + ( 1e-10, 1, CEILING), ( 1e-10, 1, UP), ( 1e-10, 0, DOWN), ( 1e-10, 0, FLOOR), + (-1e-10, 0, CEILING), (-1e-10, 0, DOWN), (-1e-10, -1, UP), (-1e-10, -1, FLOOR), # test rounding of the last nanosecond + ( 1.1234567899, 1123456790, CEILING), ( 1.1234567899, 1123456790, UP), ( 1.1234567899, 1123456789, DOWN), ( 1.1234567899, 1123456789, FLOOR), + (-1.1234567899, -1123456789, CEILING), (-1.1234567899, -1123456789, DOWN), (-1.1234567899, -1123456790, UP), (-1.1234567899, -1123456790, FLOOR), # close to 1 second + ( 0.9999999999, 1000000000, CEILING), ( 0.9999999999, 1000000000, UP), ( 0.9999999999, 999999999, DOWN), ( 0.9999999999, 999999999, FLOOR), + (-0.9999999999, -999999999, CEILING), (-0.9999999999, -999999999, DOWN), (-0.9999999999, -1000000000, UP), (-0.9999999999, -1000000000, FLOOR), @@ -890,19 +942,24 @@ class TestPyTime_t(unittest.TestCase): UP = _PyTime.ROUND_UP DOWN = _PyTime.ROUND_DOWN FLOOR = _PyTime.ROUND_FLOOR + CEILING = _PyTime.ROUND_CEILING for ns, tv, rnd in ( # nanoseconds + (1, (0, 1), CEILING), (1, (0, 1), UP), (1, (0, 0), DOWN), (1, (0, 0), FLOOR), + (-1, (0, 0), CEILING), (-1, (0, 0), DOWN), (-1, (-1, 999999), UP), (-1, (-1, 999999), FLOOR), # seconds + nanoseconds + (1234567001, (1, 234568), CEILING), (1234567001, (1, 234568), UP), (1234567001, (1, 234567), DOWN), (1234567001, (1, 234567), FLOOR), + (-1234567001, (-2, 765433), CEILING), (-1234567001, (-2, 765433), DOWN), (-1234567001, (-2, 765432), UP), (-1234567001, (-2, 765432), FLOOR), diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 128ba09..25f916b 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2635,7 +2635,7 @@ static int check_time_rounding(int round) { if (round != _PyTime_ROUND_DOWN && round != _PyTime_ROUND_UP - && round != _PyTime_ROUND_FLOOR) { + && round != _PyTime_ROUND_FLOOR && round != _PyTime_ROUND_CEILING) { PyErr_SetString(PyExc_ValueError, "invalid rounding"); return -1; } diff --git a/Python/pytime.c b/Python/pytime.c index 0d28911..ca4386a 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -31,6 +31,8 @@ _PyTime_RoundTowardsPosInf(int is_neg, _PyTime_round_t round) { if (round == _PyTime_ROUND_FLOOR) return 0; + if (round == _PyTime_ROUND_CEILING) + return 1; return ((round == _PyTime_ROUND_UP) ^ is_neg); } -- cgit v0.12 From 869e1778c0bcfc0928701c6ae0703934359d036b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 03:49:14 +0200 Subject: Issue #22117: Replace usage of _PyTime_ROUND_UP with _PyTime_ROUND_CEILING All these functions only accept positive timeouts, so this change has no effect in practice. --- Modules/_ssl.c | 4 ++-- Modules/_threadmodule.c | 7 ++++--- Modules/selectmodule.c | 7 ++++--- Modules/signalmodule.c | 3 ++- Modules/socketmodule.c | 13 +++++++------ Modules/timemodule.c | 6 +++--- 6 files changed, 22 insertions(+), 18 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 217c77c..3c909a6 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -1637,7 +1637,7 @@ check_socket_and_wait_for_timeout(PySocketSockObject *s, int writing) /* s->sock_timeout is in seconds, timeout in ms */ timeout = (int)_PyTime_AsMilliseconds(s->sock_timeout, - _PyTime_ROUND_UP); + _PyTime_ROUND_CEILING); PySSL_BEGIN_ALLOW_THREADS rc = poll(&pollfd, 1, timeout); @@ -1651,7 +1651,7 @@ check_socket_and_wait_for_timeout(PySocketSockObject *s, int writing) if (!_PyIsSelectable_fd(s->sock_fd)) return SOCKET_TOO_LARGE_FOR_SELECT; - _PyTime_AsTimeval_noraise(s->sock_timeout, &tv, _PyTime_ROUND_UP); + _PyTime_AsTimeval_noraise(s->sock_timeout, &tv, _PyTime_ROUND_CEILING); FD_ZERO(&fds); FD_SET(s->sock_fd, &fds); diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 07b01f0..0907aa0 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -59,7 +59,7 @@ acquire_timed(PyThread_type_lock lock, _PyTime_t timeout) endtime = _PyTime_GetMonotonicClock() + timeout; do { - microseconds = _PyTime_AsMicroseconds(timeout, _PyTime_ROUND_UP); + microseconds = _PyTime_AsMicroseconds(timeout, _PyTime_ROUND_CEILING); /* first a simple non-blocking try without releasing the GIL */ r = PyThread_acquire_lock_timed(lock, 0, 0); @@ -110,7 +110,8 @@ lock_acquire_parse_args(PyObject *args, PyObject *kwds, return -1; if (timeout_obj - && _PyTime_FromSecondsObject(timeout, timeout_obj, _PyTime_ROUND_UP) < 0) + && _PyTime_FromSecondsObject(timeout, + timeout_obj, _PyTime_ROUND_CEILING) < 0) return -1; if (!blocking && *timeout != unset_timeout ) { @@ -128,7 +129,7 @@ lock_acquire_parse_args(PyObject *args, PyObject *kwds, else if (*timeout != unset_timeout) { _PyTime_t microseconds; - microseconds = _PyTime_AsMicroseconds(*timeout, _PyTime_ROUND_UP); + microseconds = _PyTime_AsMicroseconds(*timeout, _PyTime_ROUND_CEILING); if (microseconds >= PY_TIMEOUT_MAX) { PyErr_SetString(PyExc_OverflowError, "timeout value is too large"); diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 2c82ce7..a852344 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -209,13 +209,13 @@ select_select(PyObject *self, PyObject *args) else { _PyTime_t ts; - if (_PyTime_FromSecondsObject(&ts, tout, _PyTime_ROUND_UP) < 0) { + if (_PyTime_FromSecondsObject(&ts, tout, _PyTime_ROUND_CEILING) < 0) { PyErr_SetString(PyExc_TypeError, "timeout must be a float or None"); return NULL; } - if (_PyTime_AsTimeval(ts, &tv, _PyTime_ROUND_UP) == -1) + if (_PyTime_AsTimeval(ts, &tv, _PyTime_ROUND_CEILING) == -1) return NULL; if (tv.tv_sec < 0) { PyErr_SetString(PyExc_ValueError, "timeout must be non-negative"); @@ -2014,7 +2014,8 @@ kqueue_queue_control(kqueue_queue_Object *self, PyObject *args) else { _PyTime_t ts; - if (_PyTime_FromSecondsObject(&ts, otimeout, _PyTime_ROUND_UP) < 0) { + if (_PyTime_FromSecondsObject(&ts, + otimeout, _PyTime_ROUND_CEILING) < 0) { PyErr_Format(PyExc_TypeError, "timeout argument must be an number " "or None, got %.200s", diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index f3b2e29..1b3589d 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -977,7 +977,8 @@ signal_sigtimedwait(PyObject *self, PyObject *args) &signals, &timeout_obj)) return NULL; - if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_UP) < 0) + if (_PyTime_FromSecondsObject(&timeout, + timeout_obj, _PyTime_ROUND_CEILING) < 0) return NULL; if (timeout < 0) { diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 513405e..a6c47ae 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -633,7 +633,7 @@ internal_select_ex(PySocketSockObject *s, int writing, _PyTime_t interval) pollfd.events = writing ? POLLOUT : POLLIN; /* s->sock_timeout is in seconds, timeout in ms */ - timeout = _PyTime_AsMilliseconds(interval, _PyTime_ROUND_UP); + timeout = _PyTime_AsMilliseconds(interval, _PyTime_ROUND_CEILING); assert(timeout <= INT_MAX); timeout_int = (int)timeout; @@ -641,7 +641,7 @@ internal_select_ex(PySocketSockObject *s, int writing, _PyTime_t interval) n = poll(&pollfd, 1, timeout_int); Py_END_ALLOW_THREADS; #else - _PyTime_AsTimeval_noraise(interval, &tv, _PyTime_ROUND_UP); + _PyTime_AsTimeval_noraise(interval, &tv, _PyTime_ROUND_CEILING); FD_ZERO(&fds); FD_SET(s->sock_fd, &fds); @@ -2191,7 +2191,8 @@ socket_parse_timeout(_PyTime_t *timeout, PyObject *timeout_obj) return 0; } - if (_PyTime_FromSecondsObject(timeout, timeout_obj, _PyTime_ROUND_UP) < 0) + if (_PyTime_FromSecondsObject(timeout, + timeout_obj, _PyTime_ROUND_CEILING) < 0) return -1; if (*timeout < 0) { @@ -2200,10 +2201,10 @@ socket_parse_timeout(_PyTime_t *timeout, PyObject *timeout_obj) } #ifdef MS_WINDOWS - overflow = (_PyTime_AsTimeval(timeout, &tv, _PyTime_ROUND_UP) < 0); + overflow = (_PyTime_AsTimeval(timeout, &tv, _PyTime_ROUND_CEILING) < 0); #endif #ifndef HAVE_POLL - timeout = _PyTime_AsMilliseconds(timeout, _PyTime_ROUND_UP); + timeout = _PyTime_AsMilliseconds(timeout, _PyTime_ROUND_CEILING); overflow = (timeout > INT_MAX); #endif if (overflow) { @@ -2452,7 +2453,7 @@ internal_connect(PySocketSockObject *s, struct sockaddr *addr, int addrlen, struct timeval tv; int conv; - _PyTime_AsTimeval_noraise(s->sock_timeout, &tv, _PyTime_ROUND_UP); + _PyTime_AsTimeval_noraise(s->sock_timeout, &tv, _PyTime_ROUND_CEILING); Py_BEGIN_ALLOW_THREADS FD_ZERO(&fds); diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 3ed3fb3..74c544a 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -221,7 +221,7 @@ static PyObject * time_sleep(PyObject *self, PyObject *obj) { _PyTime_t secs; - if (_PyTime_FromSecondsObject(&secs, obj, _PyTime_ROUND_UP)) + if (_PyTime_FromSecondsObject(&secs, obj, _PyTime_ROUND_CEILING)) return NULL; if (secs < 0) { PyErr_SetString(PyExc_ValueError, @@ -1405,7 +1405,7 @@ pysleep(_PyTime_t secs) do { #ifndef MS_WINDOWS - if (_PyTime_AsTimeval(secs, &timeout, _PyTime_ROUND_UP) < 0) + if (_PyTime_AsTimeval(secs, &timeout, _PyTime_ROUND_CEILING) < 0) return -1; Py_BEGIN_ALLOW_THREADS @@ -1420,7 +1420,7 @@ pysleep(_PyTime_t secs) return -1; } #else - millisecs = _PyTime_AsMilliseconds(secs, _PyTime_ROUND_UP); + millisecs = _PyTime_AsMilliseconds(secs, _PyTime_ROUND_CEILING); if (millisecs > (double)ULONG_MAX) { PyErr_SetString(PyExc_OverflowError, "sleep length is too large"); -- cgit v0.12 From 7d0325d6c811d66d98955e9fbf9a71c50f7e770c Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sun, 29 Mar 2015 21:53:05 -0400 Subject: #23745: handle duplicate MIME parameter names in new parser. This mimics get_param's error handling for the most part. It is slightly better in some regards as get_param can produce some really weird results for duplicate *0* parts. It departs from get_param slightly in that if we have a mix of non-extended and extended pieces for the same parameter name, the new parser assumes they were all supposed to be extended and concatenates all the values, whereas get_param always picks the non-extended parameter value. All of this error recovery is pretty much arbitrary decisions... --- Lib/email/_header_value_parser.py | 34 +++++-- Lib/test/test_email/test__header_value_parser.py | 109 +++++++++++++++++++++++ Misc/NEWS | 3 + 3 files changed, 139 insertions(+), 7 deletions(-) diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 1806cac..a9bdf44 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -71,6 +71,7 @@ import re import urllib # For urllib.parse.unquote from string import hexdigits from collections import OrderedDict +from operator import itemgetter from email import _encoded_words as _ew from email import errors from email import utils @@ -1098,15 +1099,34 @@ class MimeParameters(TokenList): params[name] = [] params[name].append((token.section_number, token)) for name, parts in params.items(): - parts = sorted(parts) - # XXX: there might be more recovery we could do here if, for - # example, this is really a case of a duplicate attribute name. + parts = sorted(parts, key=itemgetter(0)) + first_param = parts[0][1] + charset = first_param.charset + # Our arbitrary error recovery is to ignore duplicate parameters, + # to use appearance order if there are duplicate rfc 2231 parts, + # and to ignore gaps. This mimics the error recovery of get_param. + if not first_param.extended and len(parts) > 1: + if parts[1][0] == 0: + parts[1][1].defects.append(errors.InvalidHeaderDefect( + 'duplicate parameter name; duplicate(s) ignored')) + parts = parts[:1] + # Else assume the *0* was missing...note that this is different + # from get_param, but we registered a defect for this earlier. value_parts = [] - charset = parts[0][1].charset - for i, (section_number, param) in enumerate(parts): + i = 0 + for section_number, param in parts: if section_number != i: - param.defects.append(errors.InvalidHeaderDefect( - "inconsistent multipart parameter numbering")) + # We could get fancier here and look for a complete + # duplicate extended parameter and ignore the second one + # seen. But we're not doing that. The old code didn't. + if not param.extended: + param.defects.append(errors.InvalidHeaderDefect( + 'duplicate parameter name; duplicate ignored')) + continue + else: + param.defects.append(errors.InvalidHeaderDefect( + "inconsistent RFC2231 parameter numbering")) + i += 1 value = param.param_value if param.extended: try: diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 5404d19..d028f74 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -2456,6 +2456,115 @@ class TestParser(TestParserMixin, TestEmailBase): ";foo", ";foo", ";foo", [errors.InvalidHeaderDefect]*3 ) + +@parameterize +class Test_parse_mime_parameters(TestParserMixin, TestEmailBase): + + def mime_parameters_as_value(self, + value, + tl_str, + tl_value, + params, + defects): + mime_parameters = self._test_parse_x(parser.parse_mime_parameters, + value, tl_str, tl_value, defects) + self.assertEqual(mime_parameters.token_type, 'mime-parameters') + self.assertEqual(list(mime_parameters.params), params) + + + mime_parameters_params = { + + 'simple': ( + 'filename="abc.py"', + ' filename="abc.py"', + 'filename=abc.py', + [('filename', 'abc.py')], + []), + + 'multiple_keys': ( + 'filename="abc.py"; xyz=abc', + ' filename="abc.py"; xyz="abc"', + 'filename=abc.py; xyz=abc', + [('filename', 'abc.py'), ('xyz', 'abc')], + []), + + 'split_value': ( + "filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66", + ' filename="201.tif"', + "filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66", + [('filename', '201.tif')], + []), + + # Note that it is undefined what we should do for error recovery when + # there are duplicate parameter names or duplicate parts in a split + # part. We choose to ignore all duplicate parameters after the first + # and to take duplicate or missing rfc 2231 parts in apperance order. + # This is backward compatible with get_param's behavior, but the + # decisions are arbitrary. + + 'duplicate_key': ( + 'filename=abc.gif; filename=def.tiff', + ' filename="abc.gif"', + "filename=abc.gif; filename=def.tiff", + [('filename', 'abc.gif')], + [errors.InvalidHeaderDefect]), + + 'duplicate_key_with_split_value': ( + "filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66;" + " filename=abc.gif", + ' filename="201.tif"', + "filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66;" + " filename=abc.gif", + [('filename', '201.tif')], + [errors.InvalidHeaderDefect]), + + 'duplicate_key_with_split_value_other_order': ( + "filename=abc.gif; " + " filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66", + ' filename="abc.gif"', + "filename=abc.gif;" + " filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66", + [('filename', 'abc.gif')], + [errors.InvalidHeaderDefect]), + + 'duplicate_in_split_value': ( + "filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66;" + " filename*1*=abc.gif", + ' filename="201.tifabc.gif"', + "filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66;" + " filename*1*=abc.gif", + [('filename', '201.tifabc.gif')], + [errors.InvalidHeaderDefect]), + + 'missing_split_value': ( + "filename*0*=iso-8859-1''%32%30%31%2E; filename*3*=%74%69%66;", + ' filename="201.tif"', + "filename*0*=iso-8859-1''%32%30%31%2E; filename*3*=%74%69%66;", + [('filename', '201.tif')], + [errors.InvalidHeaderDefect]), + + 'duplicate_and_missing_split_value': ( + "filename*0*=iso-8859-1''%32%30%31%2E; filename*3*=%74%69%66;" + " filename*3*=abc.gif", + ' filename="201.tifabc.gif"', + "filename*0*=iso-8859-1''%32%30%31%2E; filename*3*=%74%69%66;" + " filename*3*=abc.gif", + [('filename', '201.tifabc.gif')], + [errors.InvalidHeaderDefect]*2), + + # Here we depart from get_param and assume the *0* was missing. + 'duplicate_with_broken_split_value': ( + "filename=abc.gif; " + " filename*2*=iso-8859-1''%32%30%31%2E; filename*3*=%74%69%66", + ' filename="abc.gif201.tif"', + "filename=abc.gif;" + " filename*2*=iso-8859-1''%32%30%31%2E; filename*3*=%74%69%66", + [('filename', 'abc.gif201.tif')], + # Defects are apparent missing *0*, and two 'out of sequence'. + [errors.InvalidHeaderDefect]*3), + + } + @parameterize class Test_parse_mime_version(TestParserMixin, TestEmailBase): diff --git a/Misc/NEWS b/Misc/NEWS index b80cbe2..ebac8d5 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -21,6 +21,9 @@ Core and Builtins Library ------- +- Issue #23745: The new email header parser now handles duplicate MIME + parameter names without error, similar to how get_param behaves. + - Issue #23792: Ignore KeyboardInterrupt when the pydoc pager is active. This mimics the behavior of the standard unix pagers, and prevents pipepager from shutting down while the pager itself is still running. -- cgit v0.12 From a695f83f0de060a77352174be8a5c6f6500ab98a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 03:57:14 +0200 Subject: Issue #22117: Remove _PyTime_ROUND_DOWN and _PyTime_ROUND_UP rounding methods Use _PyTime_ROUND_FLOOR and _PyTime_ROUND_CEILING instead. --- Include/pytime.h | 7 +--- Lib/test/test_time.py | 100 ++++++---------------------------------------- Modules/_testcapimodule.c | 3 +- Python/pytime.c | 20 +++------- 4 files changed, 20 insertions(+), 110 deletions(-) diff --git a/Include/pytime.h b/Include/pytime.h index ff3e203..1f14d6d 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -25,14 +25,9 @@ typedef PY_INT64_T _PyTime_t; #endif typedef enum { - /* Round towards zero. */ - _PyTime_ROUND_DOWN=0, - /* Round away from zero. - For example, used for timeout to wait "at least" N seconds. */ - _PyTime_ROUND_UP, /* Round towards minus infinity (-inf). For example, used to read a clock. */ - _PyTime_ROUND_FLOOR, + _PyTime_ROUND_FLOOR=0, /* Round towards infinity (+inf). For example, used for timeout to wait "at least" N seconds. */ _PyTime_ROUND_CEILING diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 472110c..4747cc6 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -24,20 +24,12 @@ TIME_MINYEAR = -TIME_MAXYEAR - 1 SEC_TO_NS = 10 ** 9 class _PyTime(enum.IntEnum): - # Round towards zero - ROUND_DOWN = 0 - # Round away from zero - ROUND_UP = 1 # Round towards minus infinity (-inf) - ROUND_FLOOR = 2 + ROUND_FLOOR = 0 # Round towards infinity (+inf) - ROUND_CEILING = 3 + ROUND_CEILING = 1 -ALL_ROUNDING_METHODS = ( - _PyTime.ROUND_UP, - _PyTime.ROUND_DOWN, - _PyTime.ROUND_FLOOR, - _PyTime.ROUND_CEILING) +ALL_ROUNDING_METHODS = (_PyTime.ROUND_FLOOR, _PyTime.ROUND_CEILING) class TimeTestCase(unittest.TestCase): @@ -617,13 +609,6 @@ class TestPytime(unittest.TestCase): def test_time_t(self): from _testcapi import pytime_object_to_time_t for obj, time_t, rnd in ( - # Round towards zero - (0, 0, _PyTime.ROUND_DOWN), - (-1, -1, _PyTime.ROUND_DOWN), - (-1.0, -1, _PyTime.ROUND_DOWN), - (-1.9, -1, _PyTime.ROUND_DOWN), - (1.0, 1, _PyTime.ROUND_DOWN), - (1.9, 1, _PyTime.ROUND_DOWN), # Round towards minus infinity (-inf) (0, 0, _PyTime.ROUND_FLOOR), (-1, -1, _PyTime.ROUND_FLOOR), @@ -631,13 +616,6 @@ class TestPytime(unittest.TestCase): (-1.9, -2, _PyTime.ROUND_FLOOR), (1.0, 1, _PyTime.ROUND_FLOOR), (1.9, 1, _PyTime.ROUND_FLOOR), - # Round away from zero - (0, 0, _PyTime.ROUND_UP), - (-1, -1, _PyTime.ROUND_UP), - (-1.0, -1, _PyTime.ROUND_UP), - (-1.9, -2, _PyTime.ROUND_UP), - (1.0, 1, _PyTime.ROUND_UP), - (1.9, 2, _PyTime.ROUND_UP), # Round towards infinity (+inf) (0, 0, _PyTime.ROUND_CEILING), (-1, -1, _PyTime.ROUND_CEILING), @@ -657,20 +635,6 @@ class TestPytime(unittest.TestCase): def test_timespec(self): from _testcapi import pytime_object_to_timespec for obj, timespec, rnd in ( - # Round towards zero - (0, (0, 0), _PyTime.ROUND_DOWN), - (-1, (-1, 0), _PyTime.ROUND_DOWN), - (-1.0, (-1, 0), _PyTime.ROUND_DOWN), - (1e-9, (0, 1), _PyTime.ROUND_DOWN), - (1e-10, (0, 0), _PyTime.ROUND_DOWN), - (-1e-9, (-1, 999999999), _PyTime.ROUND_DOWN), - (-1e-10, (0, 0), _PyTime.ROUND_DOWN), - (-1.2, (-2, 800000000), _PyTime.ROUND_DOWN), - (0.9999999999, (0, 999999999), _PyTime.ROUND_DOWN), - (1.1234567890, (1, 123456789), _PyTime.ROUND_DOWN), - (1.1234567899, (1, 123456789), _PyTime.ROUND_DOWN), - (-1.1234567890, (-2, 876543211), _PyTime.ROUND_DOWN), - (-1.1234567891, (-2, 876543211), _PyTime.ROUND_DOWN), # Round towards minus infinity (-inf) (0, (0, 0), _PyTime.ROUND_FLOOR), (-1, (-1, 0), _PyTime.ROUND_FLOOR), @@ -685,20 +649,6 @@ class TestPytime(unittest.TestCase): (1.1234567899, (1, 123456789), _PyTime.ROUND_FLOOR), (-1.1234567890, (-2, 876543211), _PyTime.ROUND_FLOOR), (-1.1234567891, (-2, 876543210), _PyTime.ROUND_FLOOR), - # Round away from zero - (0, (0, 0), _PyTime.ROUND_UP), - (-1, (-1, 0), _PyTime.ROUND_UP), - (-1.0, (-1, 0), _PyTime.ROUND_UP), - (1e-9, (0, 1), _PyTime.ROUND_UP), - (1e-10, (0, 1), _PyTime.ROUND_UP), - (-1e-9, (-1, 999999999), _PyTime.ROUND_UP), - (-1e-10, (-1, 999999999), _PyTime.ROUND_UP), - (-1.2, (-2, 800000000), _PyTime.ROUND_UP), - (0.9999999999, (1, 0), _PyTime.ROUND_UP), - (1.1234567890, (1, 123456790), _PyTime.ROUND_UP), - (1.1234567899, (1, 123456790), _PyTime.ROUND_UP), - (-1.1234567890, (-2, 876543211), _PyTime.ROUND_UP), - (-1.1234567891, (-2, 876543210), _PyTime.ROUND_UP), # Round towards infinity (+inf) (0, (0, 0), _PyTime.ROUND_CEILING), (-1, (-1, 0), _PyTime.ROUND_CEILING), @@ -836,40 +786,26 @@ class TestPyTime_t(unittest.TestCase): PyTime_FromSecondsObject(-9223372037.0, rnd) # Conversion giving different results depending on the rounding method - UP = _PyTime.ROUND_UP - DOWN = _PyTime.ROUND_DOWN FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING for obj, ts, rnd in ( # close to zero - ( 1e-10, 1, CEILING), - ( 1e-10, 1, UP), - ( 1e-10, 0, DOWN), ( 1e-10, 0, FLOOR), - (-1e-10, 0, CEILING), - (-1e-10, 0, DOWN), - (-1e-10, -1, UP), + ( 1e-10, 1, CEILING), (-1e-10, -1, FLOOR), + (-1e-10, 0, CEILING), # test rounding of the last nanosecond - ( 1.1234567899, 1123456790, CEILING), - ( 1.1234567899, 1123456790, UP), - ( 1.1234567899, 1123456789, DOWN), ( 1.1234567899, 1123456789, FLOOR), - (-1.1234567899, -1123456789, CEILING), - (-1.1234567899, -1123456789, DOWN), - (-1.1234567899, -1123456790, UP), + ( 1.1234567899, 1123456790, CEILING), (-1.1234567899, -1123456790, FLOOR), + (-1.1234567899, -1123456789, CEILING), # close to 1 second - ( 0.9999999999, 1000000000, CEILING), - ( 0.9999999999, 1000000000, UP), - ( 0.9999999999, 999999999, DOWN), ( 0.9999999999, 999999999, FLOOR), - (-0.9999999999, -999999999, CEILING), - (-0.9999999999, -999999999, DOWN), - (-0.9999999999, -1000000000, UP), + ( 0.9999999999, 1000000000, CEILING), (-0.9999999999, -1000000000, FLOOR), + (-0.9999999999, -999999999, CEILING), ): with self.subTest(obj=obj, round=rnd, timestamp=ts): self.assertEqual(PyTime_FromSecondsObject(obj, rnd), ts) @@ -939,30 +875,20 @@ class TestPyTime_t(unittest.TestCase): with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): self.assertEqual(PyTime_AsTimeval(ns, rnd), tv) - UP = _PyTime.ROUND_UP - DOWN = _PyTime.ROUND_DOWN FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING for ns, tv, rnd in ( # nanoseconds - (1, (0, 1), CEILING), - (1, (0, 1), UP), - (1, (0, 0), DOWN), (1, (0, 0), FLOOR), - (-1, (0, 0), CEILING), - (-1, (0, 0), DOWN), - (-1, (-1, 999999), UP), + (1, (0, 1), CEILING), (-1, (-1, 999999), FLOOR), + (-1, (0, 0), CEILING), # seconds + nanoseconds - (1234567001, (1, 234568), CEILING), - (1234567001, (1, 234568), UP), - (1234567001, (1, 234567), DOWN), (1234567001, (1, 234567), FLOOR), - (-1234567001, (-2, 765433), CEILING), - (-1234567001, (-2, 765433), DOWN), - (-1234567001, (-2, 765432), UP), + (1234567001, (1, 234568), CEILING), (-1234567001, (-2, 765432), FLOOR), + (-1234567001, (-2, 765433), CEILING), ): with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): self.assertEqual(PyTime_AsTimeval(ns, rnd), tv) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 25f916b..253efb6 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2634,8 +2634,7 @@ run_in_subinterp(PyObject *self, PyObject *args) static int check_time_rounding(int round) { - if (round != _PyTime_ROUND_DOWN && round != _PyTime_ROUND_UP - && round != _PyTime_ROUND_FLOOR && round != _PyTime_ROUND_CEILING) { + if (round != _PyTime_ROUND_FLOOR && round != _PyTime_ROUND_CEILING) { PyErr_SetString(PyExc_ValueError, "invalid rounding"); return -1; } diff --git a/Python/pytime.c b/Python/pytime.c index ca4386a..98f29ac 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -26,16 +26,6 @@ error_time_t_overflow(void) "timestamp out of range for platform time_t"); } -static int -_PyTime_RoundTowardsPosInf(int is_neg, _PyTime_round_t round) -{ - if (round == _PyTime_ROUND_FLOOR) - return 0; - if (round == _PyTime_ROUND_CEILING) - return 1; - return ((round == _PyTime_ROUND_UP) ^ is_neg); -} - time_t _PyLong_AsTime_t(PyObject *obj) { @@ -84,7 +74,7 @@ _PyTime_ObjectToDenominator(PyObject *obj, time_t *sec, long *numerator, } floatpart *= denominator; - if (_PyTime_RoundTowardsPosInf(intpart < 0, round)) { + if (round == _PyTime_ROUND_CEILING) { floatpart = ceil(floatpart); if (floatpart >= denominator) { floatpart = 0.0; @@ -121,7 +111,7 @@ _PyTime_ObjectToTime_t(PyObject *obj, time_t *sec, _PyTime_round_t round) double d, intpart, err; d = PyFloat_AsDouble(obj); - if (_PyTime_RoundTowardsPosInf(d < 0, round)) + if (round == _PyTime_ROUND_CEILING) d = ceil(d); else d = floor(d); @@ -223,7 +213,7 @@ _PyTime_FromSecondsObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round) d = PyFloat_AsDouble(obj); d *= 1e9; - if (_PyTime_RoundTowardsPosInf(d < 0, round)) + if (round == _PyTime_ROUND_CEILING) d = ceil(d); else d = floor(d); @@ -289,7 +279,7 @@ _PyTime_Multiply(_PyTime_t t, unsigned int multiply, _PyTime_round_t round) _PyTime_t k; if (multiply < SEC_TO_NS) { k = SEC_TO_NS / multiply; - if (_PyTime_RoundTowardsPosInf(t < 0, round)) + if (round == _PyTime_ROUND_CEILING) return (t + k - 1) / k; else return t / k; @@ -350,7 +340,7 @@ _PyTime_AsTimeval_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, res = -1; #endif - if (_PyTime_RoundTowardsPosInf(tv->tv_sec < 0, round)) + if (round == _PyTime_ROUND_CEILING) tv->tv_usec = (int)((ns + US_TO_NS - 1) / US_TO_NS); else tv->tv_usec = (int)(ns / US_TO_NS); -- cgit v0.12 From 7901b48a1f89b9bfa9d111ae3725400b466a9baa Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 30 Mar 2015 09:09:54 +0300 Subject: Issue #23171: csv.Writer.writerow() now supports arbitrary iterables. --- Doc/library/csv.rst | 4 ++- Lib/csv.py | 7 ++--- Lib/test/test_csv.py | 8 ++++++ Misc/NEWS | 2 ++ Modules/_csv.c | 79 ++++++++++++++++++++++++++-------------------------- 5 files changed, 54 insertions(+), 46 deletions(-) diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index e7516b6..325a421 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -419,7 +419,7 @@ Writer Objects :class:`Writer` objects (:class:`DictWriter` instances and objects returned by the :func:`writer` function) have the following public methods. A *row* must be -a sequence of strings or numbers for :class:`Writer` objects and a dictionary +an iterable of strings or numbers for :class:`Writer` objects and a dictionary mapping fieldnames to strings or numbers (by passing them through :func:`str` first) for :class:`DictWriter` objects. Note that complex numbers are written out surrounded by parens. This may cause some problems for other programs which @@ -431,6 +431,8 @@ read CSV files (assuming they support complex numbers at all). Write the *row* parameter to the writer's file object, formatted according to the current dialect. + .. versionchanged:: 3.5 + Added support of arbitrary iterables. .. method:: csvwriter.writerows(rows) diff --git a/Lib/csv.py b/Lib/csv.py index c3c31f0..ca40e5e 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -147,16 +147,13 @@ class DictWriter: if wrong_fields: raise ValueError("dict contains fields not in fieldnames: " + ", ".join([repr(x) for x in wrong_fields])) - return [rowdict.get(key, self.restval) for key in self.fieldnames] + return (rowdict.get(key, self.restval) for key in self.fieldnames) def writerow(self, rowdict): return self.writer.writerow(self._dict_to_list(rowdict)) def writerows(self, rowdicts): - rows = [] - for rowdict in rowdicts: - rows.append(self._dict_to_list(rowdict)) - return self.writer.writerows(rows) + return self.writer.writerows(map(self._dict_to_list, rowdicts)) # Guard Sniffer's type checking against builds that exclude complex() try: diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 41ef790..7be3cc3 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -186,6 +186,14 @@ class Test_Csv(unittest.TestCase): self._write_test(['a',1,'p,q'], 'a,1,p\\,q', escapechar='\\', quoting = csv.QUOTE_NONE) + def test_write_iterable(self): + self._write_test(iter(['a', 1, 'p,q']), 'a,1,"p,q"') + self._write_test(iter(['a', 1, None]), 'a,1,') + self._write_test(iter([]), '') + self._write_test(iter([None]), '""') + self._write_error_test(csv.Error, iter([None]), quoting=csv.QUOTE_NONE) + self._write_test(iter([None, None]), ',') + def test_writerows(self): class BrokenFile: def write(self, buf): diff --git a/Misc/NEWS b/Misc/NEWS index ff81056..0aae52c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -56,6 +56,8 @@ Core and Builtins Library ------- +- Issue #23171: csv.Writer.writerow() now supports arbitrary iterables. + - Issue #23745: The new email header parser now handles duplicate MIME parameter names without error, similar to how get_param behaves. diff --git a/Modules/_csv.c b/Modules/_csv.c index ade35e5..eb88626 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -1009,7 +1009,7 @@ join_reset(WriterObj *self) */ static Py_ssize_t join_append_data(WriterObj *self, unsigned int field_kind, void *field_data, - Py_ssize_t field_len, int quote_empty, int *quoted, + Py_ssize_t field_len, int *quoted, int copy_phase) { DialectObj *dialect = self->dialect; @@ -1071,18 +1071,6 @@ join_append_data(WriterObj *self, unsigned int field_kind, void *field_data, ADDCH(c); } - /* If field is empty check if it needs to be quoted. - */ - if (i == 0 && quote_empty) { - if (dialect->quoting == QUOTE_NONE) { - PyErr_Format(_csvstate_global->error_obj, - "single empty field record must be quoted"); - return -1; - } - else - *quoted = 1; - } - if (*quoted) { if (copy_phase) ADDCH(dialect->quotechar); @@ -1126,7 +1114,7 @@ join_check_rec_size(WriterObj *self, Py_ssize_t rec_len) } static int -join_append(WriterObj *self, PyObject *field, int *quoted, int quote_empty) +join_append(WriterObj *self, PyObject *field, int quoted) { unsigned int field_kind = -1; void *field_data = NULL; @@ -1141,7 +1129,7 @@ join_append(WriterObj *self, PyObject *field, int *quoted, int quote_empty) field_len = PyUnicode_GET_LENGTH(field); } rec_len = join_append_data(self, field_kind, field_data, field_len, - quote_empty, quoted, 0); + "ed, 0); if (rec_len < 0) return 0; @@ -1150,7 +1138,7 @@ join_append(WriterObj *self, PyObject *field, int *quoted, int quote_empty) return 0; self->rec_len = join_append_data(self, field_kind, field_data, field_len, - quote_empty, quoted, 1); + "ed, 1); self->num_fields++; return 1; @@ -1181,37 +1169,30 @@ join_append_lineterminator(WriterObj *self) } PyDoc_STRVAR(csv_writerow_doc, -"writerow(sequence)\n" +"writerow(iterable)\n" "\n" -"Construct and write a CSV record from a sequence of fields. Non-string\n" +"Construct and write a CSV record from an iterable of fields. Non-string\n" "elements will be converted to string."); static PyObject * csv_writerow(WriterObj *self, PyObject *seq) { DialectObj *dialect = self->dialect; - Py_ssize_t len, i; - PyObject *line, *result; + PyObject *iter, *field, *line, *result; - if (!PySequence_Check(seq)) - return PyErr_Format(_csvstate_global->error_obj, "sequence expected"); - - len = PySequence_Length(seq); - if (len < 0) - return NULL; + iter = PyObject_GetIter(seq); + if (iter == NULL) + return PyErr_Format(_csvstate_global->error_obj, + "iterable expected, not %.200s", + seq->ob_type->tp_name); /* Join all fields in internal buffer. */ join_reset(self); - for (i = 0; i < len; i++) { - PyObject *field; + while ((field = PyIter_Next(iter))) { int append_ok; int quoted; - field = PySequence_GetItem(seq, i); - if (field == NULL) - return NULL; - switch (dialect->quoting) { case QUOTE_NONNUMERIC: quoted = !PyNumber_Check(field); @@ -1225,11 +1206,11 @@ csv_writerow(WriterObj *self, PyObject *seq) } if (PyUnicode_Check(field)) { - append_ok = join_append(self, field, "ed, len == 1); + append_ok = join_append(self, field, quoted); Py_DECREF(field); } else if (field == Py_None) { - append_ok = join_append(self, NULL, "ed, len == 1); + append_ok = join_append(self, NULL, quoted); Py_DECREF(field); } else { @@ -1237,19 +1218,37 @@ csv_writerow(WriterObj *self, PyObject *seq) str = PyObject_Str(field); Py_DECREF(field); - if (str == NULL) + if (str == NULL) { + Py_DECREF(iter); return NULL; - append_ok = join_append(self, str, "ed, len == 1); + } + append_ok = join_append(self, str, quoted); Py_DECREF(str); } - if (!append_ok) + if (!append_ok) { + Py_DECREF(iter); + return NULL; + } + } + Py_DECREF(iter); + if (PyErr_Occurred()) + return NULL; + + if (self->num_fields > 0 && self->rec_size == 0) { + if (dialect->quoting == QUOTE_NONE) { + PyErr_Format(_csvstate_global->error_obj, + "single empty field record must be quoted"); + return NULL; + } + self->num_fields--; + if (!join_append(self, NULL, 1)) return NULL; } /* Add line terminator. */ if (!join_append_lineterminator(self)) - return 0; + return NULL; line = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, (void *) self->rec, self->rec_len); @@ -1261,9 +1260,9 @@ csv_writerow(WriterObj *self, PyObject *seq) } PyDoc_STRVAR(csv_writerows_doc, -"writerows(sequence of sequences)\n" +"writerows(iterable of iterables)\n" "\n" -"Construct and write a series of sequences to a csv file. Non-string\n" +"Construct and write a series of iterables to a csv file. Non-string\n" "elements will be converted to string."); static PyObject * -- cgit v0.12 From 2c7b5a9d0d6587a38f24da1f96f581bad0d3ded6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 30 Mar 2015 09:19:08 +0300 Subject: Issue #23466: %c, %o, %x, and %X in bytes formatting now raise TypeError on non-integer input. --- Lib/test/test_format.py | 19 +++++++++++- Misc/NEWS | 3 ++ Objects/bytesobject.c | 81 +++++++++++++++++++++++++++++++++---------------- 3 files changed, 76 insertions(+), 27 deletions(-) diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index e1ea33f..5a2a357 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -272,9 +272,18 @@ class FormatTest(unittest.TestCase): #test_exc(unicode('abc %\u3000','raw-unicode-escape'), 1, ValueError, # "unsupported format character '?' (0x3000) at index 5") test_exc('%d', '1', TypeError, "%d format: a number is required, not str") + test_exc('%x', '1', TypeError, "%x format: a number is required, not str") + test_exc('%x', 3.14, TypeError, "%x format: an integer is required, not float") test_exc('%g', '1', TypeError, "a float is required") test_exc('no format', '1', TypeError, "not all arguments converted during string formatting") + test_exc('%c', -1, OverflowError, "%c arg not in range(0x110000)") + test_exc('%c', sys.maxunicode+1, OverflowError, + "%c arg not in range(0x110000)") + #test_exc('%c', 2**128, OverflowError, "%c arg not in range(0x110000)") + test_exc('%c', 3.14, TypeError, "%c requires int or char") + test_exc('%c', 'ab', TypeError, "%c requires int or char") + test_exc('%c', b'x', TypeError, "%c requires int or char") if maxsize == 2**31-1: # crashes 2.2.1 and earlier: @@ -339,6 +348,8 @@ class FormatTest(unittest.TestCase): "%d format: a number is required, not str") test_exc(b'%d', b'1', TypeError, "%d format: a number is required, not bytes") + test_exc(b'%x', 3.14, TypeError, + "%x format: an integer is required, not float") test_exc(b'%g', '1', TypeError, "float argument required, not str") test_exc(b'%g', b'1', TypeError, "float argument required, not bytes") test_exc(b'no format', 7, TypeError, @@ -347,11 +358,17 @@ class FormatTest(unittest.TestCase): "not all arguments converted during bytes formatting") test_exc(b'no format', bytearray(b'1'), TypeError, "not all arguments converted during bytes formatting") + test_exc(b"%c", -1, TypeError, + "%c requires an integer in range(256) or a single byte") test_exc(b"%c", 256, TypeError, "%c requires an integer in range(256) or a single byte") + test_exc(b"%c", 2**128, TypeError, + "%c requires an integer in range(256) or a single byte") test_exc(b"%c", b"Za", TypeError, "%c requires an integer in range(256) or a single byte") - test_exc(b"%c", "Yb", TypeError, + test_exc(b"%c", "Y", TypeError, + "%c requires an integer in range(256) or a single byte") + test_exc(b"%c", 3.14, TypeError, "%c requires an integer in range(256) or a single byte") test_exc(b"%b", "Xc", TypeError, "%b requires bytes, or an object that implements __bytes__, not 'str'") diff --git a/Misc/NEWS b/Misc/NEWS index 0aae52c..36415588 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -36,6 +36,9 @@ Release date: 2015-03-28 Core and Builtins ----------------- +- Issue #23466: %c, %o, %x, and %X in bytes formatting now raise TypeError on + non-integer input. + - Issue #23573: Increased performance of string search operations (str.find, str.index, str.count, the in operator, str.split, str.partition) with arguments of different kinds (UCS1, UCS2, UCS4). diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 4d6b3e4..5a2d41c 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -433,7 +433,41 @@ formatfloat(PyObject *v, int flags, int prec, int type) return result; } -Py_LOCAL_INLINE(int) +static PyObject * +formatlong(PyObject *v, int flags, int prec, int type) +{ + PyObject *result, *iobj; + if (type == 'i') + type = 'd'; + if (PyLong_Check(v)) + return _PyUnicode_FormatLong(v, flags & F_ALT, prec, type); + if (PyNumber_Check(v)) { + /* make sure number is a type of integer for o, x, and X */ + if (type == 'o' || type == 'x' || type == 'X') + iobj = PyNumber_Index(v); + else + iobj = PyNumber_Long(v); + if (iobj == NULL) { + if (!PyErr_ExceptionMatches(PyExc_TypeError)) + return NULL; + } + else if (!PyLong_Check(iobj)) + Py_CLEAR(iobj); + if (iobj != NULL) { + result = _PyUnicode_FormatLong(iobj, flags & F_ALT, prec, type); + Py_DECREF(iobj); + return result; + } + } + PyErr_Format(PyExc_TypeError, + "%%%c format: %s is required, not %.200s", type, + (type == 'o' || type == 'x' || type == 'X') ? "an integer" + : "a number", + Py_TYPE(v)->tp_name); + return NULL; +} + +static int byte_converter(PyObject *arg, char *p) { if (PyBytes_Check(arg) && PyBytes_Size(arg) == 1) { @@ -445,12 +479,29 @@ byte_converter(PyObject *arg, char *p) return 1; } else { - long ival = PyLong_AsLong(arg); - if (0 <= ival && ival <= 255) { + PyObject *iobj; + long ival; + int overflow; + /* make sure number is a type of integer */ + if (PyLong_Check(arg)) { + ival = PyLong_AsLongAndOverflow(arg, &overflow); + } + else { + iobj = PyNumber_Index(arg); + if (iobj == NULL) { + if (!PyErr_ExceptionMatches(PyExc_TypeError)) + return 0; + goto onError; + } + ival = PyLong_AsLongAndOverflow(iobj, &overflow); + Py_DECREF(iobj); + } + if (!overflow && 0 <= ival && ival <= 255) { *p = (char)ival; return 1; } } + onError: PyErr_SetString(PyExc_TypeError, "%c requires an integer in range(256) or a single byte"); return 0; @@ -561,7 +612,6 @@ _PyBytes_Format(PyObject *format, PyObject *args) int prec = -1; int c = '\0'; int fill; - PyObject *iobj; PyObject *v = NULL; PyObject *temp = NULL; const char *pbuf = NULL; @@ -747,28 +797,7 @@ _PyBytes_Format(PyObject *format, PyObject *args) case 'o': case 'x': case 'X': - if (c == 'i') - c = 'd'; - iobj = NULL; - if (PyNumber_Check(v)) { - if ((PyLong_Check(v))) { - iobj = v; - Py_INCREF(iobj); - } - else { - iobj = PyNumber_Long(v); - if (iobj != NULL && !PyLong_Check(iobj)) - Py_CLEAR(iobj); - } - } - if (iobj == NULL) { - PyErr_Format(PyExc_TypeError, - "%%%c format: a number is required, " - "not %.200s", c, Py_TYPE(v)->tp_name); - goto error; - } - temp = _PyUnicode_FormatLong(iobj, flags & F_ALT, prec, c); - Py_DECREF(iobj); + temp = formatlong(v, flags, prec, c); if (!temp) goto error; assert(PyUnicode_IS_ASCII(temp)); -- cgit v0.12 From 4aa867959f19ce92ccf999aa9f9dc4bd0b49ead7 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 30 Mar 2015 09:52:29 +0300 Subject: Issue #23783: Fixed memory leak in PyObject_ClearWeakRefs() in case of MemoryError. --- Objects/weakrefobject.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index c083f8f..d4d52e6 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -900,11 +900,9 @@ PyObject_ClearWeakRefs(PyObject *object) if (*list != NULL) { PyWeakReference *current = *list; Py_ssize_t count = _PyWeakref_GetWeakrefCount(current); - int restore_error = PyErr_Occurred() ? 1 : 0; PyObject *err_type, *err_value, *err_tb; - if (restore_error) - PyErr_Fetch(&err_type, &err_value, &err_tb); + PyErr_Fetch(&err_type, &err_value, &err_tb); if (count == 1) { PyObject *callback = current->wr_callback; @@ -922,8 +920,7 @@ PyObject_ClearWeakRefs(PyObject *object) tuple = PyTuple_New(count * 2); if (tuple == NULL) { - if (restore_error) - PyErr_Fetch(&err_type, &err_value, &err_tb); + _PyErr_ChainExceptions(err_type, err_value, err_tb); return; } @@ -954,7 +951,7 @@ PyObject_ClearWeakRefs(PyObject *object) } Py_DECREF(tuple); } - if (restore_error) - PyErr_Restore(err_type, err_value, err_tb); + assert(!PyErr_Occurred()); + PyErr_Restore(err_type, err_value, err_tb); } } -- cgit v0.12 From 04d09ebd396d2a87ad7204e170c1149edc9b141d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 30 Mar 2015 09:58:41 +0300 Subject: Issue #23785: Fixed memory leak in TextIOWrapper.tell() in rare circumstances. --- Modules/_io/textio.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 3f01980..d1c0d01 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -2435,14 +2435,10 @@ fail: if (saved_state) { PyObject *type, *value, *traceback; PyErr_Fetch(&type, &value, &traceback); - res = _PyObject_CallMethodId(self->decoder, &PyId_setstate, "(O)", saved_state); + _PyErr_ChainExceptions(type, value, traceback); Py_DECREF(saved_state); - if (res == NULL) - return NULL; - Py_DECREF(res); - - PyErr_Restore(type, value, traceback); + Py_XDECREF(res); } return NULL; } -- cgit v0.12 From e134a7fe36652434c2ccffc4ebab2ec2031d1505 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 10:09:31 +0200 Subject: Issue #23752: _Py_fstat() is now responsible to raise the Python exception Add _Py_fstat_noraise() function when a Python exception is not welcome. --- Include/fileutils.h | 8 ++++-- Modules/_io/fileio.c | 10 +++----- Modules/main.c | 6 +++-- Modules/mmapmodule.c | 25 +++++++++---------- Modules/posixmodule.c | 2 +- Modules/signalmodule.c | 10 +++----- Programs/_freeze_importlib.c | 6 ++--- Python/dynload_shlib.c | 14 +++++------ Python/fileutils.c | 58 +++++++++++++++++++++++++++++++++++--------- Python/marshal.c | 2 +- Python/random.c | 3 +-- Python/sysmodule.c | 2 +- 12 files changed, 89 insertions(+), 57 deletions(-) diff --git a/Include/fileutils.h b/Include/fileutils.h index 93a9297..4fd3172 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -41,12 +41,16 @@ struct _Py_stat_struct { PyAPI_FUNC(int) _Py_fstat( int fd, - struct _Py_stat_struct *stat); + struct _Py_stat_struct *status); + +PyAPI_FUNC(int) _Py_fstat_noraise( + int fd, + struct _Py_stat_struct *status); #endif /* Py_LIMITED_API */ PyAPI_FUNC(int) _Py_stat( PyObject *path, - struct stat *statbuf); + struct stat *status); #ifndef Py_LIMITED_API PyAPI_FUNC(int) _Py_open( diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 595f99e..b56a9c3 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -399,10 +399,8 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) } self->blksize = DEFAULT_BUFFER_SIZE; - if (_Py_fstat(self->fd, &fdfstat) < 0) { - PyErr_SetFromErrno(PyExc_OSError); + if (_Py_fstat(self->fd, &fdfstat) < 0) goto error; - } #if defined(S_ISDIR) && defined(EISDIR) /* On Unix, open will succeed for directories. In Python, there should be no file objects referring to @@ -589,7 +587,7 @@ new_buffersize(fileio *self, size_t currentsize) static PyObject * fileio_readall(fileio *self) { - struct _Py_stat_struct st; + struct _Py_stat_struct status; Py_off_t pos, end; PyObject *result; Py_ssize_t bytes_read = 0; @@ -606,8 +604,8 @@ fileio_readall(fileio *self) #else pos = lseek(self->fd, 0L, SEEK_CUR); #endif - if (_Py_fstat(self->fd, &st) == 0) - end = st.st_size; + if (_Py_fstat_noraise(self->fd, &status) == 0) + end = status.st_size; else end = (Py_off_t)-1; diff --git a/Modules/main.c b/Modules/main.c index 74e512b..2a9ea28 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -753,9 +753,11 @@ Py_Main(int argc, wchar_t **argv) } { struct _Py_stat_struct sb; - if (_Py_fstat(fileno(fp), &sb) == 0 && + if (_Py_fstat_noraise(fileno(fp), &sb) == 0 && S_ISDIR(sb.st_mode)) { - fprintf(stderr, "%ls: '%ls' is a directory, cannot continue\n", argv[0], filename); + fprintf(stderr, + "%ls: '%ls' is a directory, cannot continue\n", + argv[0], filename); fclose(fp); return 1; } diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 25056a4..e2ed5f9 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -465,15 +465,13 @@ mmap_size_method(mmap_object *self, #ifdef UNIX { - struct _Py_stat_struct buf; - if (-1 == _Py_fstat(self->fd, &buf)) { - PyErr_SetFromErrno(PyExc_OSError); + struct _Py_stat_struct status; + if (_Py_fstat(self->fd, &status) == -1) return NULL; - } #ifdef HAVE_LARGEFILE_SUPPORT - return PyLong_FromLongLong(buf.st_size); + return PyLong_FromLongLong(status.st_size); #else - return PyLong_FromLong(buf.st_size); + return PyLong_FromLong(status.st_size); #endif } #endif /* UNIX */ @@ -1112,7 +1110,7 @@ _GetMapSize(PyObject *o, const char* param) static PyObject * new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) { - struct _Py_stat_struct st; + struct _Py_stat_struct status; mmap_object *m_obj; PyObject *map_size_obj = NULL; Py_ssize_t map_size; @@ -1177,25 +1175,26 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) if (fd != -1) (void)fcntl(fd, F_FULLFSYNC); #endif - if (fd != -1 && _Py_fstat(fd, &st) == 0 && S_ISREG(st.st_mode)) { + if (fd != -1 && _Py_fstat_noraise(fd, &status) == 0 + && S_ISREG(status.st_mode)) { if (map_size == 0) { - if (st.st_size == 0) { + if (status.st_size == 0) { PyErr_SetString(PyExc_ValueError, "cannot mmap an empty file"); return NULL; } - if (offset >= st.st_size) { + if (offset >= status.st_size) { PyErr_SetString(PyExc_ValueError, "mmap offset is greater than file size"); return NULL; } - if (st.st_size - offset > PY_SSIZE_T_MAX) { + if (status.st_size - offset > PY_SSIZE_T_MAX) { PyErr_SetString(PyExc_ValueError, "mmap length is too large"); return NULL; } - map_size = (Py_ssize_t) (st.st_size - offset); - } else if (offset + map_size > st.st_size) { + map_size = (Py_ssize_t) (status.st_size - offset); + } else if (offset + map_size > status.st_size) { PyErr_SetString(PyExc_ValueError, "mmap length is greater than file size"); return NULL; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 801305f..ef69a45 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -351,7 +351,7 @@ static int win32_can_symlink = 0; #ifdef MS_WINDOWS # define STAT win32_stat # define LSTAT win32_lstat -# define FSTAT _Py_fstat +# define FSTAT _Py_fstat_noraise # define STRUCT_STAT struct _Py_stat_struct #else # define STAT stat diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 1b3589d..3081562 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -503,7 +503,7 @@ signal_siginterrupt(PyObject *self, PyObject *args) static PyObject * signal_set_wakeup_fd(PyObject *self, PyObject *args) { - struct _Py_stat_struct st; + struct _Py_stat_struct status; #ifdef MS_WINDOWS PyObject *fdobj; SOCKET_T sockfd, old_sockfd; @@ -559,10 +559,8 @@ signal_set_wakeup_fd(PyObject *self, PyObject *args) return NULL; } - if (_Py_fstat(fd, &st) != 0) { - PyErr_SetExcFromWindowsErr(PyExc_OSError, GetLastError()); + if (_Py_fstat(fd, &status) != 0) return NULL; - } /* on Windows, a file cannot be set to non-blocking mode */ } @@ -591,10 +589,8 @@ signal_set_wakeup_fd(PyObject *self, PyObject *args) return NULL; } - if (_Py_fstat(fd, &st) != 0) { - PyErr_SetFromErrno(PyExc_OSError); + if (_Py_fstat(fd, &status) != 0) return NULL; - } blocking = _Py_get_blocking(fd); if (blocking < 0) diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_importlib.c index 31b3d31..b72fcf4 100644 --- a/Programs/_freeze_importlib.c +++ b/Programs/_freeze_importlib.c @@ -35,7 +35,7 @@ main(int argc, char *argv[]) { char *inpath, *outpath; FILE *infile = NULL, *outfile = NULL; - struct _Py_stat_struct st; + struct _Py_stat_struct status; size_t text_size, data_size, n; char *text = NULL; unsigned char *data; @@ -54,11 +54,11 @@ main(int argc, char *argv[]) fprintf(stderr, "cannot open '%s' for reading\n", inpath); goto error; } - if (_Py_fstat(fileno(infile), &st)) { + if (_Py_fstat_noraise(fileno(infile), &status)) { fprintf(stderr, "cannot fstat '%s'\n", inpath); goto error; } - text_size = st.st_size; + text_size = status.st_size; text = (char *) malloc(text_size + 1); if (text == NULL) { fprintf(stderr, "could not allocate %ld bytes\n", (long) text_size); diff --git a/Python/dynload_shlib.c b/Python/dynload_shlib.c index 659adac..1a467fd 100644 --- a/Python/dynload_shlib.c +++ b/Python/dynload_shlib.c @@ -71,22 +71,20 @@ dl_funcptr _PyImport_GetDynLoadFunc(const char *shortname, if (fp != NULL) { int i; - struct _Py_stat_struct statb; - if (_Py_fstat(fileno(fp), &statb) == -1) { - PyErr_SetFromErrno(PyExc_IOError); + struct _Py_stat_struct status; + if (_Py_fstat(fileno(fp), &status) == -1) return NULL; - } for (i = 0; i < nhandles; i++) { - if (statb.st_dev == handles[i].dev && - statb.st_ino == handles[i].ino) { + if (status.st_dev == handles[i].dev && + status.st_ino == handles[i].ino) { p = (dl_funcptr) dlsym(handles[i].handle, funcname); return p; } } if (nhandles < 128) { - handles[nhandles].dev = statb.st_dev; - handles[nhandles].ino = statb.st_ino; + handles[nhandles].dev = status.st_dev; + handles[nhandles].ino = status.st_ino; } } diff --git a/Python/fileutils.c b/Python/fileutils.c index e6d3154..daaad2a 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -565,7 +565,8 @@ attributes_to_mode(DWORD attr) } void -_Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag, struct _Py_stat_struct *result) +_Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag, + struct _Py_stat_struct *result) { memset(result, 0, sizeof(*result)); result->st_mode = attributes_to_mode(info->dwFileAttributes); @@ -595,9 +596,12 @@ _Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag, files larger than 2 GB. fstat() may fail with EOVERFLOW on files larger than 2 GB because the file size type is an signed 32-bit integer: see issue #23152. - */ + + On Windows, set the last Windows error and return nonzero on error. On + POSIX, set errno and return nonzero on error. Fill status and return 0 on + success. */ int -_Py_fstat(int fd, struct _Py_stat_struct *result) +_Py_fstat_noraise(int fd, struct _Py_stat_struct *status) { #ifdef MS_WINDOWS BY_HANDLE_FILE_INFORMATION info; @@ -619,22 +623,21 @@ _Py_fstat(int fd, struct _Py_stat_struct *result) SetLastError(ERROR_INVALID_HANDLE); return -1; } - memset(result, 0, sizeof(*result)); + memset(status, 0, sizeof(*status)); type = GetFileType(h); if (type == FILE_TYPE_UNKNOWN) { DWORD error = GetLastError(); - if (error != 0) { + if (error != 0) return -1; - } /* else: valid but unknown file */ } if (type != FILE_TYPE_DISK) { if (type == FILE_TYPE_CHAR) - result->st_mode = _S_IFCHR; + status->st_mode = _S_IFCHR; else if (type == FILE_TYPE_PIPE) - result->st_mode = _S_IFIFO; + status->st_mode = _S_IFIFO; return 0; } @@ -642,15 +645,48 @@ _Py_fstat(int fd, struct _Py_stat_struct *result) return -1; } - _Py_attribute_data_to_stat(&info, 0, result); + _Py_attribute_data_to_stat(&info, 0, status); /* specific to fstat() */ - result->st_ino = (((__int64)info.nFileIndexHigh)<<32) + info.nFileIndexLow; + status->st_ino = (((__int64)info.nFileIndexHigh)<<32) + info.nFileIndexLow; return 0; #else - return fstat(fd, result); + return fstat(fd, status); #endif } +/* Return information about a file. + + On POSIX, use fstat(). + + On Windows, use GetFileType() and GetFileInformationByHandle() which support + files larger than 2 GB. fstat() may fail with EOVERFLOW on files larger + than 2 GB because the file size type is an signed 32-bit integer: see issue + #23152. + + Raise an exception and return -1 on error. On Windows, set the last Windows + error on error. On POSIX, set errno on error. Fill status and return 0 on + success. + + The GIL must be held. */ +int +_Py_fstat(int fd, struct _Py_stat_struct *status) +{ + int res; + + Py_BEGIN_ALLOW_THREADS + res = _Py_fstat_noraise(fd, status); + Py_END_ALLOW_THREADS + + if (res != 0) { +#ifdef MS_WINDOWS + PyErr_SetFromWindowsErr(0); +#else + PyErr_SetFromErrno(PyExc_OSError); +#endif + return -1; + } + return 0; +} /* Call _wstat() on Windows, or encode the path to the filesystem encoding and call stat() otherwise. Only fill st_mode attribute on Windows. diff --git a/Python/marshal.c b/Python/marshal.c index 3472882..f89cd04 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -1486,7 +1486,7 @@ static off_t getfilesize(FILE *fp) { struct _Py_stat_struct st; - if (_Py_fstat(fileno(fp), &st) != 0) + if (_Py_fstat_noraise(fileno(fp), &st) != 0) return -1; #if SIZEOF_OFF_T == 4 else if (st.st_size >= INT_MAX) diff --git a/Python/random.c b/Python/random.c index a281829..a4eba3c 100644 --- a/Python/random.c +++ b/Python/random.c @@ -221,7 +221,7 @@ dev_urandom_python(char *buffer, Py_ssize_t size) if (urandom_cache.fd >= 0) { /* Does the fd point to the same thing as before? (issue #21207) */ - if (_Py_fstat(urandom_cache.fd, &st) + if (_Py_fstat_noraise(urandom_cache.fd, &st) || st.st_dev != urandom_cache.st_dev || st.st_ino != urandom_cache.st_ino) { /* Something changed: forget the cached fd (but don't close it, @@ -250,7 +250,6 @@ dev_urandom_python(char *buffer, Py_ssize_t size) } else { if (_Py_fstat(fd, &st)) { - PyErr_SetFromErrno(PyExc_OSError); close(fd); return -1; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 471389c..9ec2521 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1690,7 +1690,7 @@ _PySys_Init(void) #if !defined(MS_WINDOWS) { struct _Py_stat_struct sb; - if (_Py_fstat(fileno(stdin), &sb) == 0 && + if (_Py_fstat_noraise(fileno(stdin), &sb) == 0 && S_ISDIR(sb.st_mode)) { /* There's nothing more we can do. */ /* Py_FatalError() will core dump, so just exit. */ -- cgit v0.12 From 45cff0c0e6c4a31ed3b5b88ee803320862fbd43a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 10:22:16 +0200 Subject: Issue #22117: Try to fix rounding in conversion from Python double to _PyTime_t using the C volatile keyword. --- Python/pytime.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/pytime.c b/Python/pytime.c index 98f29ac..5bf8c56 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -207,7 +207,8 @@ int _PyTime_FromSecondsObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round) { if (PyFloat_Check(obj)) { - double d, err; + /* volatile avoids unsafe optimization on float enabled by gcc -O3 */ + volatile double d, err; /* convert to a number of nanoseconds */ d = PyFloat_AsDouble(obj); -- cgit v0.12