From f0604fddc3b9a775f22d19d7d7fe50c4aa1ecf7d Mon Sep 17 00:00:00 2001 From: Richard Oudkerk Date: Mon, 11 Jun 2012 17:56:08 +0100 Subject: Issue #3518: Remove references to non-existent BaseManager.from_address() method --- Doc/library/multiprocessing.rst | 7 ++++--- Lib/multiprocessing/managers.py | 4 ---- Misc/NEWS | 3 +++ 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 43ae086..5e56ecb 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1236,9 +1236,10 @@ their parent process exits. The manager classes are defined in the type of shared object. This must be a string. *callable* is a callable used for creating objects for this type - identifier. If a manager instance will be created using the - :meth:`from_address` classmethod or if the *create_method* argument is - ``False`` then this can be left as ``None``. + identifier. If a manager instance will be connected to the + server using the :meth:`connect` method, or if the + *create_method* argument is ``False`` then this can be left as + ``None``. *proxytype* is a subclass of :class:`BaseProxy` which is used to create proxies for shared objects with this *typeid*. If ``None`` then a proxy diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 6a7dccb..7059095 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -455,10 +455,6 @@ class BaseManager(object): self._serializer = serializer self._Listener, self._Client = listener_client[serializer] - def __reduce__(self): - return type(self).from_address, \ - (self._address, self._authkey, self._serializer) - def get_server(self): ''' Return server object with serve_forever() method and address attribute diff --git a/Misc/NEWS b/Misc/NEWS index 13a1320..4012c96 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -21,6 +21,9 @@ Core and Builtins Library ------- +- Issue #3518: Remove references to non-existent BaseManager.from_address() + method. + - Issue #13857: Added textwrap.indent() function (initial patch by Ezra Berch) -- cgit v0.12 From b7832939c752038299ea2f6acbb9789a9aaff707 Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Tue, 12 Jun 2012 21:06:06 +0200 Subject: 1) Fix signature of _mpd_qpow_uint(): contrary to the comment base is constant. 2) Abort the loop for all specials, not only infinity. 3) Make the function more general and distinguish between zero clamping and folding down the exponent. The latter case is currently handled by setting context->clamp to 0 before calling the function. --- Modules/_decimal/libmpdec/mpdecimal.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Modules/_decimal/libmpdec/mpdecimal.c b/Modules/_decimal/libmpdec/mpdecimal.c index ff6d867..262e834 100644 --- a/Modules/_decimal/libmpdec/mpdecimal.c +++ b/Modules/_decimal/libmpdec/mpdecimal.c @@ -107,8 +107,9 @@ static inline void _mpd_qmul(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); static void _mpd_base_ndivmod(mpd_t *q, mpd_t *r, const mpd_t *a, const mpd_t *b, uint32_t *status); -static inline void _mpd_qpow_uint(mpd_t *result, mpd_t *base, mpd_uint_t exp, - uint8_t resultsign, const mpd_context_t *ctx, uint32_t *status); +static inline void _mpd_qpow_uint(mpd_t *result, const mpd_t *base, + mpd_uint_t exp, uint8_t resultsign, + const mpd_context_t *ctx, uint32_t *status); mpd_uint_t mpd_qsshiftr(mpd_t *result, const mpd_t *a, mpd_ssize_t n); @@ -5841,12 +5842,12 @@ mpd_qnext_toward(mpd_t *result, const mpd_t *a, const mpd_t *b, } /* - * Internal function: Integer power with mpd_uint_t exponent, base is modified! - * Function can fail with MPD_Malloc_error. + * Internal function: Integer power with mpd_uint_t exponent. The function + * can fail with MPD_Malloc_error. */ static inline void -_mpd_qpow_uint(mpd_t *result, mpd_t *base, mpd_uint_t exp, uint8_t resultsign, - const mpd_context_t *ctx, uint32_t *status) +_mpd_qpow_uint(mpd_t *result, const mpd_t *base, mpd_uint_t exp, + uint8_t resultsign, const mpd_context_t *ctx, uint32_t *status) { uint32_t workstatus = 0; mpd_uint_t n; @@ -5866,7 +5867,8 @@ _mpd_qpow_uint(mpd_t *result, mpd_t *base, mpd_uint_t exp, uint8_t resultsign, if (exp & n) { mpd_qmul(result, result, base, ctx, &workstatus); } - if (workstatus & (MPD_Overflow|MPD_Clamped)) { + if (mpd_isspecial(result) || + (mpd_iszerocoeff(result) && (workstatus & MPD_Clamped))) { break; } } -- cgit v0.12 From d9738242f82109e01561131efecb2e20da93d743 Mon Sep 17 00:00:00 2001 From: Alexander Belopolsky Date: Tue, 12 Jun 2012 16:14:17 -0400 Subject: Fixed a typo in time_localtime() --- Modules/timemodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 5961ac9..a80cb4b 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -401,7 +401,7 @@ time_localtime(PyObject *self, PyObject *args) if (!parse_time_t_args(args, "|O:localtime", &when)) return NULL; - if (pylocaltime(&when, &buf) == 1) + if (pylocaltime(&when, &buf) == -1) return NULL; return tmtotuple(&buf); } -- cgit v0.12 From bda4b8802c9693cd1c6f97cc09e34c4f63b030ef Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 12 Jun 2012 22:11:44 +0200 Subject: time.get_clock_info() uses a namespace instead of structseq --- Doc/library/time.rst | 36 ++++++++++----------------------- Modules/timemodule.c | 57 ++++++++++++++++++---------------------------------- 2 files changed, 31 insertions(+), 62 deletions(-) diff --git a/Doc/library/time.rst b/Doc/library/time.rst index a8c70b3..ba6b176 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -160,30 +160,6 @@ The module defines the following functions and data items: .. versionadded:: 3.3 -.. class:: clock_info - - Clock information object returned by :func:`get_clock_info`. - - .. attribute:: implementation - - The name of the underlying C function used to get the clock value. - - .. attribute:: monotonic - - ``True`` if the clock cannot go backward, ``False`` otherwise. - - .. attribute:: adjusted - - ``True`` if the clock can be adjusted (e.g. by a NTP daemon), ``False`` - otherwise. - - .. attribute:: resolution - - The resolution of the clock in seconds (:class:`float`). - - .. versionadded:: 3.3 - - .. function:: clock_settime(clk_id, time) Set the time of the specified clock *clk_id*. @@ -267,7 +243,7 @@ The module defines the following functions and data items: .. function:: get_clock_info(name) - Get information on the specified clock as a :class:`clock_info` object. + Get information on the specified clock as a namespace object. Supported clock names and the corresponding functions to read their value are: @@ -277,6 +253,16 @@ The module defines the following functions and data items: * ``'process_time'``: :func:`time.process_time` * ``'time'``: :func:`time.time` + The result has the following attributes: + + - *adjusted*: ``True`` if the clock can be adjusted (e.g. by a NTP daemon), + ``False`` otherwise + - *implementation*: The name of the underlying C function used to get + the clock value + - *monotonic*: ``True`` if the clock cannot go backward, + ``False`` otherwise + - *resolution*: The resolution of the clock in seconds (:class:`float`) + .. versionadded:: 3.3 diff --git a/Modules/timemodule.c b/Modules/timemodule.c index a80cb4b..d844bc3 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -1124,35 +1124,12 @@ PyDoc_STRVAR(process_time_doc, Process time for profiling: sum of the kernel and user-space CPU time."); -static PyTypeObject ClockInfoType; - -PyDoc_STRVAR(ClockInfo_docstring, - "Clock information"); - -static PyStructSequence_Field ClockInfo_fields[] = { - {"implementation", "name of the underlying C function " - "used to get the clock value"}, - {"monotonic", "True if the clock cannot go backward, False otherwise"}, - {"adjusted", "True if the clock can be adjusted " - "(e.g. by a NTP daemon), False otherwise"}, - {"resolution", "resolution of the clock in seconds"}, - {NULL, NULL} -}; - -static PyStructSequence_Desc ClockInfo_desc = { - "time.clock_info", - ClockInfo_docstring, - ClockInfo_fields, - 4, -}; - static PyObject * time_get_clock_info(PyObject *self, PyObject *args) { char *name; - PyObject *obj; _Py_clock_info_t info; - PyObject *result; + PyObject *obj = NULL, *dict, *ns; if (!PyArg_ParseTuple(args, "s:get_clock_info", &name)) return NULL; @@ -1191,39 +1168,50 @@ time_get_clock_info(PyObject *self, PyObject *args) return NULL; Py_DECREF(obj); - result = PyStructSequence_New(&ClockInfoType); - if (result == NULL) + dict = PyDict_New(); + if (dict == NULL) return NULL; assert(info.implementation != NULL); obj = PyUnicode_FromString(info.implementation); if (obj == NULL) goto error; - PyStructSequence_SET_ITEM(result, 0, obj); + if (PyDict_SetItemString(dict, "implementation", obj) == -1) + goto error; + Py_CLEAR(obj); assert(info.monotonic != -1); obj = PyBool_FromLong(info.monotonic); if (obj == NULL) goto error; - PyStructSequence_SET_ITEM(result, 1, obj); + if (PyDict_SetItemString(dict, "monotonic", obj) == -1) + goto error; + Py_CLEAR(obj); assert(info.adjusted != -1); obj = PyBool_FromLong(info.adjusted); if (obj == NULL) goto error; - PyStructSequence_SET_ITEM(result, 2, obj); + if (PyDict_SetItemString(dict, "adjusted", obj) == -1) + goto error; + Py_CLEAR(obj); assert(info.resolution > 0.0); assert(info.resolution <= 1.0); obj = PyFloat_FromDouble(info.resolution); if (obj == NULL) goto error; - PyStructSequence_SET_ITEM(result, 3, obj); + if (PyDict_SetItemString(dict, "resolution", obj) == -1) + goto error; + Py_CLEAR(obj); - return result; + ns = _PyNamespace_New(dict); + Py_DECREF(dict); + return ns; error: - Py_DECREF(result); + Py_DECREF(dict); + Py_XDECREF(obj); return NULL; } @@ -1451,11 +1439,6 @@ PyInit_time(void) PyStructSequence_InitType(&StructTimeType, &struct_time_type_desc); - /* initialize ClockInfoType */ - PyStructSequence_InitType(&ClockInfoType, &ClockInfo_desc); - Py_INCREF(&ClockInfoType); - PyModule_AddObject(m, "clock_info", (PyObject*)&ClockInfoType); - #ifdef MS_WINDOWS winver.dwOSVersionInfoSize = sizeof(winver); if (!GetVersionEx((OSVERSIONINFO*)&winver)) { -- cgit v0.12 From 2b89fdf7eb17b5ca3928aa3c2f6552e79386c677 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 12 Jun 2012 22:46:37 +0200 Subject: PEP 418: Rename adjusted attribute to adjustable in time.get_clock_info() result Fix also its value on Windows and Linux according to its documentation: "adjustable" indicates if the clock *can be* adjusted, not if it is or was adjusted. In most cases, it is not possible to indicate if a clock is or was adjusted. --- Doc/library/time.rst | 4 ++-- Include/pytime.h | 2 +- Lib/test/test_time.py | 12 ++++++------ Misc/NEWS | 2 ++ Modules/timemodule.c | 36 +++++++++++++++--------------------- Python/pytime.c | 11 ++++------- 6 files changed, 30 insertions(+), 37 deletions(-) diff --git a/Doc/library/time.rst b/Doc/library/time.rst index ba6b176..2a765ac 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -255,8 +255,8 @@ The module defines the following functions and data items: The result has the following attributes: - - *adjusted*: ``True`` if the clock can be adjusted (e.g. by a NTP daemon), - ``False`` otherwise + - *adjustable*: ``True`` if the clock can be changed automatically (e.g. by + a NTP daemon) or manually by the system administrator, ``False`` otherwise - *implementation*: The name of the underlying C function used to get the clock value - *monotonic*: ``True`` if the clock cannot go backward, diff --git a/Include/pytime.h b/Include/pytime.h index dd4cc69..52902f5 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -26,7 +26,7 @@ typedef struct { typedef struct { const char *implementation; int monotonic; - int adjusted; + int adjustable; double resolution; } _Py_clock_info_t; diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 02f05c3..bd72af5 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -32,14 +32,14 @@ class TimeTestCase(unittest.TestCase): info = time.get_clock_info('time') self.assertFalse(info.monotonic) if sys.platform != 'win32': - self.assertTrue(info.adjusted) + self.assertTrue(info.adjustable) def test_clock(self): time.clock() info = time.get_clock_info('clock') self.assertTrue(info.monotonic) - self.assertFalse(info.adjusted) + self.assertFalse(info.adjustable) @unittest.skipUnless(hasattr(time, 'clock_gettime'), 'need time.clock_gettime()') @@ -372,9 +372,9 @@ class TimeTestCase(unittest.TestCase): info = time.get_clock_info('monotonic') self.assertTrue(info.monotonic) if sys.platform == 'linux': - self.assertTrue(info.adjusted) + self.assertTrue(info.adjustable) else: - self.assertFalse(info.adjusted) + self.assertFalse(info.adjustable) def test_perf_counter(self): time.perf_counter() @@ -390,7 +390,7 @@ class TimeTestCase(unittest.TestCase): info = time.get_clock_info('process_time') self.assertTrue(info.monotonic) - self.assertFalse(info.adjusted) + self.assertFalse(info.adjustable) @unittest.skipUnless(hasattr(time, 'monotonic'), 'need time.monotonic') @@ -441,7 +441,7 @@ class TimeTestCase(unittest.TestCase): # 0.0 < resolution <= 1.0 self.assertGreater(info.resolution, 0.0) self.assertLessEqual(info.resolution, 1.0) - self.assertIsInstance(info.adjusted, bool) + self.assertIsInstance(info.adjustable, bool) self.assertRaises(ValueError, time.get_clock_info, 'xxx') diff --git a/Misc/NEWS b/Misc/NEWS index 4012c96..e47766a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -21,6 +21,8 @@ Core and Builtins Library ------- +- Rename adjusted attribute to adjustable in time.get_clock_info() result. + - Issue #3518: Remove references to non-existent BaseManager.from_address() method. diff --git a/Modules/timemodule.c b/Modules/timemodule.c index d844bc3..0a9c431 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -96,7 +96,7 @@ floatclock(_Py_clock_info_t *info) info->implementation = "clock()"; info->resolution = 1.0 / (double)CLOCKS_PER_SEC; info->monotonic = 1; - info->adjusted = 0; + info->adjustable = 0; } return PyFloat_FromDouble((double)value / CLOCKS_PER_SEC); } @@ -132,7 +132,7 @@ win_perf_counter(_Py_clock_info_t *info, PyObject **result) info->implementation = "QueryPerformanceCounter()"; info->resolution = 1.0 / (double)cpu_frequency; info->monotonic = 1; - info->adjusted = 0; + info->adjustable = 0; } *result = PyFloat_FromDouble(diff / (double)cpu_frequency); return 0; @@ -882,7 +882,7 @@ pymonotonic(_Py_clock_info_t *info) return NULL; } info->resolution = timeIncrement * 1e-7; - info->adjusted = 0; + info->adjustable = 0; } return PyFloat_FromDouble(result); @@ -903,7 +903,7 @@ pymonotonic(_Py_clock_info_t *info) info->implementation = "mach_absolute_time()"; info->resolution = (double)timebase.numer / timebase.denom * 1e-9; info->monotonic = 1; - info->adjusted = 0; + info->adjustable = 0; } return PyFloat_FromDouble(secs); @@ -926,13 +926,7 @@ pymonotonic(_Py_clock_info_t *info) struct timespec res; info->monotonic = 1; info->implementation = function; -#if (defined(linux) || defined(__linux) || defined(__linux__)) \ - && !defined(CLOCK_HIGHRES) - /* CLOCK_MONOTONIC is adjusted on Linux */ - info->adjusted = 1; -#else - info->adjusted = 0; -#endif + info->adjustable = 0; if (clock_getres(clk_id, &res) == 0) info->resolution = res.tv_sec + res.tv_nsec * 1e-9; else @@ -1024,7 +1018,7 @@ py_process_time(_Py_clock_info_t *info) info->implementation = "GetProcessTimes()"; info->resolution = 1e-7; info->monotonic = 1; - info->adjusted = 0; + info->adjustable = 0; } return PyFloat_FromDouble(total * 1e-7); #else @@ -1053,7 +1047,7 @@ py_process_time(_Py_clock_info_t *info) struct timespec res; info->implementation = function; info->monotonic = 1; - info->adjusted = 0; + info->adjustable = 0; if (clock_getres(clk_id, &res) == 0) info->resolution = res.tv_sec + res.tv_nsec * 1e-9; else @@ -1071,7 +1065,7 @@ py_process_time(_Py_clock_info_t *info) if (info) { info->implementation = "getrusage(RUSAGE_SELF)"; info->monotonic = 1; - info->adjusted = 0; + info->adjustable = 0; info->resolution = 1e-6; } return PyFloat_FromDouble(total); @@ -1100,7 +1094,7 @@ py_process_time(_Py_clock_info_t *info) if (info) { info->implementation = "times()"; info->monotonic = 1; - info->adjusted = 0; + info->adjustable = 0; info->resolution = 1.0 / ticks_per_second; } return PyFloat_FromDouble(total); @@ -1137,12 +1131,12 @@ time_get_clock_info(PyObject *self, PyObject *args) #ifdef Py_DEBUG info.implementation = NULL; info.monotonic = -1; - info.adjusted = -1; + info.adjustable = -1; info.resolution = -1.0; #else info.implementation = ""; info.monotonic = 0; - info.adjusted = 0; + info.adjustable = 0; info.resolution = 1.0; #endif @@ -1188,11 +1182,11 @@ time_get_clock_info(PyObject *self, PyObject *args) goto error; Py_CLEAR(obj); - assert(info.adjusted != -1); - obj = PyBool_FromLong(info.adjusted); + assert(info.adjustable != -1); + obj = PyBool_FromLong(info.adjustable); if (obj == NULL) goto error; - if (PyDict_SetItemString(dict, "adjusted", obj) == -1) + if (PyDict_SetItemString(dict, "adjustable", obj) == -1) goto error; Py_CLEAR(obj); @@ -1471,7 +1465,7 @@ floattime(_Py_clock_info_t *info) struct timespec res; info->implementation = "clock_gettime(CLOCK_REALTIME)"; info->monotonic = 0; - info->adjusted = 1; + info->adjustable = 1; if (clock_getres(CLOCK_REALTIME, &res) == 0) info->resolution = res.tv_sec + res.tv_nsec * 1e-9; else diff --git a/Python/pytime.c b/Python/pytime.c index eb5685b..beeab87 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -44,10 +44,7 @@ pygettimeofday(_PyTime_timeval *tp, _Py_clock_info_t *info) (void) GetSystemTimeAdjustment(&timeAdjustment, &timeIncrement, &isTimeAdjustmentDisabled); info->resolution = timeIncrement * 1e-7; - if (isTimeAdjustmentDisabled) - info->adjusted = 0; - else - info->adjusted = 1; + info->adjustable = 1; } #else /* There are three ways to get the time: @@ -71,7 +68,7 @@ pygettimeofday(_PyTime_timeval *tp, _Py_clock_info_t *info) info->implementation = "gettimeofday()"; info->resolution = 1e-6; info->monotonic = 0; - info->adjusted = 1; + info->adjustable = 1; } return; } @@ -87,7 +84,7 @@ pygettimeofday(_PyTime_timeval *tp, _Py_clock_info_t *info) info->implementation = "ftime()"; info->resolution = 1e-3; info->monotonic = 0; - info->adjusted = 1; + info->adjustable = 1; } } #else /* !HAVE_FTIME */ @@ -97,7 +94,7 @@ pygettimeofday(_PyTime_timeval *tp, _Py_clock_info_t *info) info->implementation = "time()"; info->resolution = 1.0; info->monotonic = 0; - info->adjusted = 1; + info->adjustable = 1; } #endif /* !HAVE_FTIME */ -- cgit v0.12 From 6222d76952c42ec6b7134b51bf242d41a0b972bd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 12 Jun 2012 23:04:11 +0200 Subject: Fix test_time for adjusted/adjustable changes --- Lib/test/test_time.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index bd72af5..9ea8f0c 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -31,8 +31,7 @@ class TimeTestCase(unittest.TestCase): time.time() info = time.get_clock_info('time') self.assertFalse(info.monotonic) - if sys.platform != 'win32': - self.assertTrue(info.adjustable) + self.assertTrue(info.adjustable) def test_clock(self): time.clock() @@ -371,10 +370,7 @@ class TimeTestCase(unittest.TestCase): info = time.get_clock_info('monotonic') self.assertTrue(info.monotonic) - if sys.platform == 'linux': - self.assertTrue(info.adjustable) - else: - self.assertFalse(info.adjustable) + self.assertFalse(info.adjustable) def test_perf_counter(self): time.perf_counter() -- cgit v0.12 From 56692f572231be88ea95d9165156d2b2d6033a56 Mon Sep 17 00:00:00 2001 From: Sandro Tosi Date: Wed, 13 Jun 2012 23:58:54 +0200 Subject: Issue #15060: fix typo in socket doc; Patch by anatoly techtonik --- Doc/library/socket.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index f236d30..532f0d3 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -47,7 +47,7 @@ Socket addresses are represented as follows: - A pair ``(host, port)`` is used for the :const:`AF_INET` address family, where *host* is a string representing either a hostname in Internet domain notation like ``'daring.cwi.nl'`` or an IPv4 address like ``'100.50.200.5'``, - and *port* is an integral port number. + and *port* is an integer port number. - For :const:`AF_INET6` address family, a four-tuple ``(host, port, flowinfo, scopeid)`` is used, where *flowinfo* and *scopeid* represent the ``sin6_flowinfo`` -- cgit v0.12 From 27b130e7026843de85954c4c976ec96c0e226987 Mon Sep 17 00:00:00 2001 From: Sandro Tosi Date: Thu, 14 Jun 2012 00:37:09 +0200 Subject: Issue #15060: better fix, thanks to review on #python-dev --- Doc/library/socket.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 532f0d3..0aef183 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -47,7 +47,7 @@ Socket addresses are represented as follows: - A pair ``(host, port)`` is used for the :const:`AF_INET` address family, where *host* is a string representing either a hostname in Internet domain notation like ``'daring.cwi.nl'`` or an IPv4 address like ``'100.50.200.5'``, - and *port* is an integer port number. + and *port* is an integer. - For :const:`AF_INET6` address family, a four-tuple ``(host, port, flowinfo, scopeid)`` is used, where *flowinfo* and *scopeid* represent the ``sin6_flowinfo`` -- cgit v0.12 From c142bba2a7ead80b99aef44b3dca06e2a4be4923 Mon Sep 17 00:00:00 2001 From: Alexander Belopolsky Date: Wed, 13 Jun 2012 22:15:26 -0400 Subject: Issue #1667546: On platforms supporting tm_zone and tm_gmtoff fields in struct tm, time.struct_time objects returned by time.gmtime(), time.localtime() and time.strptime() functions now have tm_zone and tm_gmtoff attributes. Original patch by Paul Boddie. --- Doc/library/time.rst | 23 ++++++++++++++++++-- Lib/_strptime.py | 6 +++--- Lib/test/test_structseq.py | 5 +++-- Lib/test/test_time.py | 53 +++++++++++++++++++++++++++++++++++++++++++++- Misc/NEWS | 5 +++++ Modules/timemodule.c | 36 +++++++++++++++++++++++++++++-- 6 files changed, 118 insertions(+), 10 deletions(-) diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 2a765ac..3faabf7 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -77,6 +77,12 @@ An explanation of some terminology and conventions is in order. See :class:`struct_time` for a description of these objects. + .. versionchanged:: 3.3 + + The :class:`struct_time` type was extended to provide the + :attr:`tm_gmtoff` and :attr:`tm_zone` attributes when platform + supports corresponding ``struct tm`` members. + * Use the following functions to convert between time representations: +-------------------------+-------------------------+-------------------------+ @@ -336,7 +342,6 @@ The module defines the following functions and data items: .. versionadded:: 3.3 - .. function:: sleep(secs) Suspend execution for the given number of seconds. The argument may be a @@ -433,6 +438,12 @@ The module defines the following functions and data items: | ``%Y`` | Year with century as a decimal number. | | | | | | +-----------+------------------------------------------------+-------+ + | ``%z`` | Time zone offset indicating a positive or | | + | | negative time difference from UTC/GMT of the | | + | | form +HHMM or -HHMM, where H represents decimal| | + | | hour digits and M represents decimal minute | | + | | digits [-23:59, +23:59]. | | + +-----------+------------------------------------------------+-------+ | ``%Z`` | Time zone name (no characters if no time zone | | | | exists). | | +-----------+------------------------------------------------+-------+ @@ -532,6 +543,10 @@ The module defines the following functions and data items: +-------+-------------------+---------------------------------+ | 8 | :attr:`tm_isdst` | 0, 1 or -1; see below | +-------+-------------------+---------------------------------+ + | N/A | :attr:`tm_zone` | abbreviation of timezone name | + +-------+-------------------+---------------------------------+ + | N/A | :attr:`tm_gmtoff` | offset from UTC in seconds | + +-------+-------------------+---------------------------------+ Note that unlike the C structure, the month value is a range of [1, 12], not [0, 11]. A ``-1`` argument as the daylight @@ -542,6 +557,11 @@ The module defines the following functions and data items: :class:`struct_time`, or having elements of the wrong type, a :exc:`TypeError` is raised. + .. versionchanged:: 3.3 + + :attr:`tm_gmtoff` and :attr:`tm_zone` attributes are avaliable on + platforms with C library supporting the corresponding fields in + ``struct tm``. .. function:: time() @@ -552,7 +572,6 @@ The module defines the following functions and data items: lower value than a previous call if the system clock has been set back between the two calls. - .. data:: timezone The offset of the local (non-DST) timezone, in seconds west of UTC (negative in diff --git a/Lib/_strptime.py b/Lib/_strptime.py index fa06376..b0cd3d6 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -486,19 +486,19 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): return (year, month, day, hour, minute, second, - weekday, julian, tz, gmtoff, tzname), fraction + weekday, julian, tz, tzname, gmtoff), fraction def _strptime_time(data_string, format="%a %b %d %H:%M:%S %Y"): """Return a time struct based on the input string and the format string.""" tt = _strptime(data_string, format)[0] - return time.struct_time(tt[:9]) + return time.struct_time(tt[:time._STRUCT_TM_ITEMS]) def _strptime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"): """Return a class cls instance based on the input string and the format string.""" tt, fraction = _strptime(data_string, format) - gmtoff, tzname = tt[-2:] + tzname, gmtoff = tt[-2:] args = tt[:6] + (fraction,) if gmtoff is not None: tzdelta = datetime_timedelta(seconds=gmtoff) diff --git a/Lib/test/test_structseq.py b/Lib/test/test_structseq.py index d6c63b7..a89e955 100644 --- a/Lib/test/test_structseq.py +++ b/Lib/test/test_structseq.py @@ -78,8 +78,9 @@ class StructSeqTest(unittest.TestCase): def test_fields(self): t = time.gmtime() - self.assertEqual(len(t), t.n_fields) - self.assertEqual(t.n_fields, t.n_sequence_fields+t.n_unnamed_fields) + self.assertEqual(len(t), t.n_sequence_fields) + self.assertEqual(t.n_unnamed_fields, 0) + self.assertEqual(t.n_fields, time._STRUCT_TM_ITEMS) def test_constructor(self): t = time.struct_time diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 9ea8f0c..63e1453 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -620,7 +620,58 @@ class TestPytime(unittest.TestCase): for invalid in self.invalid_values: self.assertRaises(OverflowError, pytime_object_to_timespec, invalid) - + @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") + def test_localtime_timezone(self): + + # Get the localtime and examine it for the offset and zone. + lt = time.localtime() + self.assertTrue(hasattr(lt, "tm_gmtoff")) + self.assertTrue(hasattr(lt, "tm_zone")) + + # See if the offset and zone are similar to the module + # attributes. + if lt.tm_gmtoff is None: + self.assertTrue(not hasattr(time, "timezone")) + else: + self.assertEqual(lt.tm_gmtoff, -[time.timezone, time.altzone][lt.tm_isdst]) + if lt.tm_zone is None: + self.assertTrue(not hasattr(time, "tzname")) + else: + self.assertEqual(lt.tm_zone, time.tzname[lt.tm_isdst]) + + # Try and make UNIX times from the localtime and a 9-tuple + # created from the localtime. Test to see that the times are + # the same. + t = time.mktime(lt); t9 = time.mktime(lt[:9]) + self.assertEqual(t, t9) + + # Make localtimes from the UNIX times and compare them to + # the original localtime, thus making a round trip. + new_lt = time.localtime(t); new_lt9 = time.localtime(t9) + self.assertEqual(new_lt, lt) + self.assertEqual(new_lt.tm_gmtoff, lt.tm_gmtoff) + self.assertEqual(new_lt.tm_zone, lt.tm_zone) + self.assertEqual(new_lt9, lt) + self.assertEqual(new_lt.tm_gmtoff, lt.tm_gmtoff) + self.assertEqual(new_lt9.tm_zone, lt.tm_zone) + + @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") + def test_strptime_timezone(self): + t = time.strptime("UTC", "%Z") + self.assertEqual(t.tm_zone, 'UTC') + t = time.strptime("+0500", "%z") + self.assertEqual(t.tm_gmtoff, 5 * 3600) + + @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") + def test_short_times(self): + + import pickle + + # Load a short time structure using pickle. + st = b"ctime\nstruct_time\np0\n((I2007\nI8\nI11\nI1\nI24\nI49\nI5\nI223\nI1\ntp1\n(dp2\ntp3\nRp4\n." + lt = pickle.loads(st) + self.assertIs(lt.tm_gmtoff, None) + self.assertIs(lt.tm_zone, None) def test_main(): support.run_unittest( diff --git a/Misc/NEWS b/Misc/NEWS index e47766a..f3b886c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -21,6 +21,11 @@ Core and Builtins Library ------- +- Issue #1667546: On platforms supporting tm_zone and tm_gmtoff fields + in struct tm, time.struct_time objects returned by time.gmtime(), + time.localtime() and time.strptime() functions now have tm_zone and + tm_gmtoff attributes. Original patch by Paul Boddie. + - Rename adjusted attribute to adjustable in time.get_clock_info() result. - Issue #3518: Remove references to non-existent BaseManager.from_address() diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 0a9c431..161407d 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -275,6 +275,10 @@ static PyStructSequence_Field struct_time_type_fields[] = { {"tm_wday", "day of week, range [0, 6], Monday is 0"}, {"tm_yday", "day of year, range [1, 366]"}, {"tm_isdst", "1 if summer time is in effect, 0 if not, and -1 if unknown"}, +#ifdef HAVE_STRUCT_TM_TM_ZONE + {"tm_zone", "abbreviation of timezone name"}, + {"tm_gmtoff", "offset from UTC in seconds"}, +#endif /* HAVE_STRUCT_TM_TM_ZONE */ {0} }; @@ -294,6 +298,7 @@ static PyStructSequence_Desc struct_time_type_desc = { static int initialized; static PyTypeObject StructTimeType; + static PyObject * tmtotuple(struct tm *p) { @@ -312,6 +317,11 @@ tmtotuple(struct tm *p) SET(6, (p->tm_wday + 6) % 7); /* Want Monday == 0 */ SET(7, p->tm_yday + 1); /* Want January, 1 == 1 */ SET(8, p->tm_isdst); +#ifdef HAVE_STRUCT_TM_TM_ZONE + PyStructSequence_SET_ITEM(v, 9, + PyUnicode_DecodeLocale(p->tm_zone, "surrogateescape")); + SET(10, p->tm_gmtoff); +#endif /* HAVE_STRUCT_TM_TM_ZONE */ #undef SET if (PyErr_Occurred()) { Py_XDECREF(v); @@ -371,7 +381,10 @@ PyDoc_STRVAR(gmtime_doc, tm_sec, tm_wday, tm_yday, tm_isdst)\n\ \n\ Convert seconds since the Epoch to a time tuple expressing UTC (a.k.a.\n\ -GMT). When 'seconds' is not passed in, convert the current time instead."); +GMT). When 'seconds' is not passed in, convert the current time instead.\n\ +\n\ +If the platform supports the tm_gmtoff and tm_zone, they are available as\n\ +attributes only."); static int pylocaltime(time_t *timep, struct tm *result) @@ -438,6 +451,17 @@ gettmarg(PyObject *args, struct tm *p) p->tm_mon--; p->tm_wday = (p->tm_wday + 1) % 7; p->tm_yday--; +#ifdef HAVE_STRUCT_TM_TM_ZONE + if (Py_TYPE(args) == &StructTimeType) { + PyObject *item; + item = PyTuple_GET_ITEM(args, 9); + p->tm_zone = item == Py_None ? NULL : _PyUnicode_AsString(item); + item = PyTuple_GET_ITEM(args, 10); + p->tm_gmtoff = item == Py_None ? 0 : PyLong_AsLong(item); + if (PyErr_Occurred()) + return 0; + } +#endif /* HAVE_STRUCT_TM_TM_ZONE */ return 1; } @@ -778,7 +802,10 @@ time_mktime(PyObject *self, PyObject *tup) PyDoc_STRVAR(mktime_doc, "mktime(tuple) -> floating point number\n\ \n\ -Convert a time tuple in local time to seconds since the Epoch."); +Convert a time tuple in local time to seconds since the Epoch.\n\ +Note that mktime(gmtime(0)) will not generally return zero for most\n\ +time zones; instead the returned value will either be equal to that\n\ +of the timezone or altzone attributes on the time module."); #endif /* HAVE_MKTIME */ #ifdef HAVE_WORKING_TZSET @@ -1443,6 +1470,11 @@ PyInit_time(void) #endif } Py_INCREF(&StructTimeType); +#ifdef HAVE_STRUCT_TM_TM_ZONE + PyModule_AddIntConstant(m, "_STRUCT_TM_ITEMS", 11); +#else + PyModule_AddIntConstant(m, "_STRUCT_TM_ITEMS", 9); +#endif PyModule_AddObject(m, "struct_time", (PyObject*) &StructTimeType); initialized = 1; return m; -- cgit v0.12 From 993fe3f0357b3539f770c64ae249809981069345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Thu, 14 Jun 2012 15:37:21 +0200 Subject: Issue #14937: Fix typo. Patch by Roger Serwy. --- Lib/idlelib/AutoComplete.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/idlelib/AutoComplete.py b/Lib/idlelib/AutoComplete.py index 6b5391b..e4c1aff 100644 --- a/Lib/idlelib/AutoComplete.py +++ b/Lib/idlelib/AutoComplete.py @@ -143,7 +143,7 @@ class AutoComplete: elif hp.is_in_code() and (not mode or mode==COMPLETE_ATTRIBUTES): self._remove_autocomplete_window() mode = COMPLETE_ATTRIBUTES - while i and curline[i-1] in ID_CHARS or ord(curline[i-1]) > 127: + while i and (curline[i-1] in ID_CHARS or ord(curline[i-1]) > 127): i -= 1 comp_start = curline[i:j] if i and curline[i-1] == '.': -- cgit v0.12 From c838ec1f821e7a79838f37c45119ce028e7c4940 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Thu, 14 Jun 2012 16:00:24 +0200 Subject: Issue #14936: curses_panel was converted to PEP 3121 API. Patch by Robin Schreiber. --- Misc/NEWS | 3 +++ Modules/_curses_panel.c | 54 ++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index f3b886c..ffac575 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -21,6 +21,9 @@ Core and Builtins Library ------- +- Issue #14936: curses_panel was converted to PEP 3121 API. + Patch by Robin Schreiber. + - Issue #1667546: On platforms supporting tm_zone and tm_gmtoff fields in struct tm, time.struct_time objects returned by time.gmtime(), time.localtime() and time.strptime() functions now have tm_zone and diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index 8561a2e..7d773d3 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -16,8 +16,38 @@ static char *PyCursesVersion = "2.1"; #include -static PyObject *PyCursesError; +typedef struct { + PyObject *PyCursesError; +} _curses_panelstate; + +#define _curses_panelstate(o) ((_curses_panelstate *)PyModule_GetState(o)) + +/*static PyObject *PyCursesError;*/ + +static int +_curses_panel_clear(PyObject *m) +{ + Py_CLEAR(_curses_panelstate(m)->PyCursesError); + return 0; +} + +static int +_curses_panel_traverse(PyObject *m, visitproc visit, void *arg) +{ + Py_VISIT(_curses_panelstate(m)->PyCursesError); + return 0; +} + +static void +_curses_panel_free(void *m) +{ + _curses_panel_clear((PyObject *) m); +} + +static struct PyModuleDef _curses_panelmodule; +#define _curses_panelstate_global \ +((_curses_panelstate *) PyModule_GetState(PyState_FindModule(&_curses_panelmodule))) /* Utility Functions */ @@ -34,9 +64,9 @@ PyCursesCheckERR(int code, char *fname) return Py_None; } else { if (fname == NULL) { - PyErr_SetString(PyCursesError, catchall_ERR); + PyErr_SetString(_curses_panelstate_global->PyCursesError, catchall_ERR); } else { - PyErr_Format(PyCursesError, "%s() returned ERR", fname); + PyErr_Format(_curses_panelstate_global->PyCursesError, "%s() returned ERR", fname); } return NULL; } @@ -280,7 +310,7 @@ PyCursesPanel_replace_panel(PyCursesPanelObject *self, PyObject *args) rtn = replace_panel(self->pan, temp->win); if (rtn == ERR) { - PyErr_SetString(PyCursesError, "replace_panel() returned ERR"); + PyErr_SetString(_curses_panelstate_global->PyCursesError, "replace_panel() returned ERR"); return NULL; } Py_DECREF(po->wo); @@ -305,7 +335,7 @@ PyCursesPanel_userptr(PyCursesPanelObject *self) PyCursesInitialised; obj = (PyObject *) panel_userptr(self->pan); if (obj == NULL) { - PyErr_SetString(PyCursesError, "no userptr set"); + PyErr_SetString(_curses_panelstate_global->PyCursesError, "no userptr set"); return NULL; } @@ -405,7 +435,7 @@ PyCurses_new_panel(PyObject *self, PyObject *args) return NULL; pan = new_panel(win->win); if (pan == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + PyErr_SetString(_curses_panelstate_global->PyCursesError, catchall_NULL); return NULL; } return (PyObject *)PyCursesPanel_New(pan, win); @@ -467,12 +497,12 @@ static struct PyModuleDef _curses_panelmodule = { PyModuleDef_HEAD_INIT, "_curses_panel", NULL, - -1, + sizeof(_curses_panelstate), PyCurses_methods, NULL, - NULL, - NULL, - NULL + _curses_panel_traverse, + _curses_panel_clear, + _curses_panel_free }; PyMODINIT_FUNC @@ -493,8 +523,8 @@ PyInit__curses_panel(void) d = PyModule_GetDict(m); /* For exception _curses_panel.error */ - PyCursesError = PyErr_NewException("_curses_panel.error", NULL, NULL); - PyDict_SetItemString(d, "error", PyCursesError); + _curses_panelstate(m)->PyCursesError = PyErr_NewException("_curses_panel.error", NULL, NULL); + PyDict_SetItemString(d, "error", _curses_panelstate(m)->PyCursesError); /* Make the version available */ v = PyUnicode_FromString(PyCursesVersion); -- cgit v0.12 From bc07cb883e5c03b8c108c2c9d86bc0a158d62c27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Thu, 14 Jun 2012 16:01:23 +0200 Subject: Issue #14936: curses_panel was converted to PEP 3121 and PEP 384 API. Patch by Robin Schreiber. --- Misc/NEWS | 2 +- Modules/_curses_panel.c | 72 ++++++++++++++++++++----------------------------- 2 files changed, 30 insertions(+), 44 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index ffac575..bfb9f80 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -21,7 +21,7 @@ Core and Builtins Library ------- -- Issue #14936: curses_panel was converted to PEP 3121 API. +- Issue #14936: curses_panel was converted to PEP 3121 and PEP 384 API. Patch by Robin Schreiber. - Issue #1667546: On platforms supporting tm_zone and tm_gmtoff fields diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index 7d773d3..1c7d084 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -18,12 +18,11 @@ static char *PyCursesVersion = "2.1"; typedef struct { PyObject *PyCursesError; + PyObject *PyCursesPanel_Type; } _curses_panelstate; #define _curses_panelstate(o) ((_curses_panelstate *)PyModule_GetState(o)) -/*static PyObject *PyCursesError;*/ - static int _curses_panel_clear(PyObject *m) { @@ -84,9 +83,8 @@ typedef struct { PyCursesWindowObject *wo; /* for reference counts */ } PyCursesPanelObject; -PyTypeObject PyCursesPanel_Type; - -#define PyCursesPanel_Check(v) (Py_TYPE(v) == &PyCursesPanel_Type) +#define PyCursesPanel_Check(v) \ + (Py_TYPE(v) == _curses_panelstate_global->PyCursesPanel_Type) /* Some helper functions. The problem is that there's always a window associated with a panel. To ensure that Python's GC doesn't pull @@ -205,7 +203,8 @@ PyCursesPanel_New(PANEL *pan, PyCursesWindowObject *wo) { PyCursesPanelObject *po; - po = PyObject_NEW(PyCursesPanelObject, &PyCursesPanel_Type); + po = PyObject_NEW(PyCursesPanelObject, + (PyTypeObject *)(_curses_panelstate_global)->PyCursesPanel_Type); if (po == NULL) return NULL; po->pan = pan; if (insert_lop(po) < 0) { @@ -364,36 +363,18 @@ static PyMethodDef PyCursesPanel_Methods[] = { /* -------------------------------------------------------*/ -PyTypeObject PyCursesPanel_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_curses_panel.curses panel", /*tp_name*/ - sizeof(PyCursesPanelObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - /* methods */ - (destructor)PyCursesPanel_Dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_reserved*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - PyCursesPanel_Methods, /*tp_methods*/ +static PyType_Slot PyCursesPanel_Type_slots[] = { + {Py_tp_dealloc, PyCursesPanel_Dealloc}, + {Py_tp_methods, PyCursesPanel_Methods}, + {0, 0}, +}; + +static PyType_Spec PyCursesPanel_Type_spec = { + "_curses_panel.curses panel", + sizeof(PyCursesPanelObject), + 0, + Py_TPFLAGS_DEFAULT, + PyCursesPanel_Type_slots }; /* Wrapper for panel_above(NULL). This function returns the bottom @@ -510,18 +491,20 @@ PyInit__curses_panel(void) { PyObject *m, *d, *v; - /* Initialize object type */ - if (PyType_Ready(&PyCursesPanel_Type) < 0) - return NULL; - - import_curses(); - /* Create the module and add the functions */ m = PyModule_Create(&_curses_panelmodule); if (m == NULL) - return NULL; + goto fail; d = PyModule_GetDict(m); + /* Initialize object type */ + _curses_panelstate(m)->PyCursesPanel_Type = \ + PyType_FromSpec(&PyCursesPanel_Type_spec); + if (_curses_panelstate(m)->PyCursesPanel_Type == NULL) + goto fail; + + import_curses(); + /* For exception _curses_panel.error */ _curses_panelstate(m)->PyCursesError = PyErr_NewException("_curses_panel.error", NULL, NULL); PyDict_SetItemString(d, "error", _curses_panelstate(m)->PyCursesError); @@ -532,4 +515,7 @@ PyInit__curses_panel(void) PyDict_SetItemString(d, "__version__", v); Py_DECREF(v); return m; + fail: + Py_XDECREF(m); + return NULL; } -- cgit v0.12 From 73d9a292aeed5aa3414c911c1f34afeea17f0dda Mon Sep 17 00:00:00 2001 From: Richard Oudkerk Date: Thu, 14 Jun 2012 15:30:10 +0100 Subject: Issue #13841: Make child processes exit using sys.exit() on Windows --- Lib/multiprocessing/forking.py | 6 ++-- Lib/multiprocessing/managers.py | 67 +++++++++++++++++----------------------- Lib/multiprocessing/util.py | 27 +++++++++------- Lib/test/support.py | 2 +- Lib/test/test_multiprocessing.py | 5 +++ Misc/NEWS | 2 ++ 6 files changed, 54 insertions(+), 55 deletions(-) diff --git a/Lib/multiprocessing/forking.py b/Lib/multiprocessing/forking.py index 3a474cd..4baf548 100644 --- a/Lib/multiprocessing/forking.py +++ b/Lib/multiprocessing/forking.py @@ -13,7 +13,7 @@ import signal from multiprocessing import util, process -__all__ = ['Popen', 'assert_spawning', 'exit', 'duplicate', 'close', 'ForkingPickler'] +__all__ = ['Popen', 'assert_spawning', 'duplicate', 'close', 'ForkingPickler'] # # Check that the current thread is spawning a child process @@ -75,7 +75,6 @@ else: # if sys.platform != 'win32': - exit = os._exit duplicate = os.dup close = os.close @@ -168,7 +167,6 @@ else: WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False)) WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") - exit = _winapi.ExitProcess close = _winapi.CloseHandle # @@ -349,7 +347,7 @@ else: from_parent.close() exitcode = self._bootstrap() - exit(exitcode) + sys.exit(exitcode) def get_preparation_data(name): diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 7059095..817d232 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -22,7 +22,7 @@ import queue from traceback import format_exc from multiprocessing import Process, current_process, active_children, Pool, util, connection from multiprocessing.process import AuthenticationString -from multiprocessing.forking import exit, Popen, ForkingPickler +from multiprocessing.forking import Popen, ForkingPickler from time import time as _time # @@ -140,28 +140,38 @@ class Server(object): self.id_to_obj = {'0': (None, ())} self.id_to_refcount = {} self.mutex = threading.RLock() - self.stop = 0 def serve_forever(self): ''' Run the server forever ''' + self.stop_event = threading.Event() current_process()._manager_server = self try: + accepter = threading.Thread(target=self.accepter) + accepter.daemon = True + accepter.start() try: - while 1: - try: - c = self.listener.accept() - except (OSError, IOError): - continue - t = threading.Thread(target=self.handle_request, args=(c,)) - t.daemon = True - t.start() + while not self.stop_event.is_set(): + self.stop_event.wait(1) except (KeyboardInterrupt, SystemExit): pass finally: - self.stop = 999 - self.listener.close() + if sys.stdout != sys.__stdout__: + util.debug('resetting stdout, stderr') + sys.stdout = sys.__stdout__ + sys.stderr = sys.__stderr__ + sys.exit(0) + + def accepter(self): + while True: + try: + c = self.listener.accept() + except (OSError, IOError): + continue + t = threading.Thread(target=self.handle_request, args=(c,)) + t.daemon = True + t.start() def handle_request(self, c): ''' @@ -208,7 +218,7 @@ class Server(object): send = conn.send id_to_obj = self.id_to_obj - while not self.stop: + while not self.stop_event.is_set(): try: methodname = obj = None @@ -318,32 +328,13 @@ class Server(object): Shutdown this process ''' try: - try: - util.debug('manager received shutdown message') - c.send(('#RETURN', None)) - - if sys.stdout != sys.__stdout__: - util.debug('resetting stdout, stderr') - sys.stdout = sys.__stdout__ - sys.stderr = sys.__stderr__ - - util._run_finalizers(0) - - for p in active_children(): - util.debug('terminating a child process of manager') - p.terminate() - - for p in active_children(): - util.debug('terminating a child process of manager') - p.join() - - util._run_finalizers() - util.info('manager exiting with exitcode 0') - except: - import traceback - traceback.print_exc() + util.debug('manager received shutdown message') + c.send(('#RETURN', None)) + except: + import traceback + traceback.print_exc() finally: - exit(0) + self.stop_event.set() def create(self, c, typeid, *args, **kwds): ''' diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 48abe38..8a6aede 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -269,21 +269,24 @@ _exiting = False def _exit_function(): global _exiting - info('process shutting down') - debug('running all "atexit" finalizers with priority >= 0') - _run_finalizers(0) + if not _exiting: + _exiting = True - for p in active_children(): - if p._daemonic: - info('calling terminate() for daemon %s', p.name) - p._popen.terminate() + info('process shutting down') + debug('running all "atexit" finalizers with priority >= 0') + _run_finalizers(0) - for p in active_children(): - info('calling join() for process %s', p.name) - p.join() + for p in active_children(): + if p._daemonic: + info('calling terminate() for daemon %s', p.name) + p._popen.terminate() - debug('running the remaining "atexit" finalizers') - _run_finalizers() + for p in active_children(): + info('calling join() for process %s', p.name) + p.join() + + debug('running the remaining "atexit" finalizers') + _run_finalizers() atexit.register(_exit_function) diff --git a/Lib/test/support.py b/Lib/test/support.py index 6749a511..3ff1df5 100644 --- a/Lib/test/support.py +++ b/Lib/test/support.py @@ -1593,7 +1593,7 @@ def strip_python_stderr(stderr): This will typically be run on the result of the communicate() method of a subprocess.Popen object. """ - stderr = re.sub(br"\[\d+ refs\]\r?\n?$", b"", stderr).strip() + stderr = re.sub(br"\[\d+ refs\]\r?\n?", b"", stderr).strip() return stderr def args_from_interpreter_flags(): diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py index 65e7b0b..e4ad904 100644 --- a/Lib/test/test_multiprocessing.py +++ b/Lib/test/test_multiprocessing.py @@ -1564,6 +1564,11 @@ class _TestMyManager(BaseTestCase): manager.shutdown() + # If the manager process exited cleanly then the exitcode + # will be zero. Otherwise (after a short timeout) + # terminate() is used, resulting in an exitcode of -SIGTERM. + self.assertEqual(manager._process.exitcode, 0) + # # Test of connecting to a remote server and using xmlrpclib for serialization # diff --git a/Misc/NEWS b/Misc/NEWS index bfb9f80..058555f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -21,6 +21,8 @@ Core and Builtins Library ------- +- Issue #13841: Make child processes exit using sys.exit() on Windows. + - Issue #14936: curses_panel was converted to PEP 3121 and PEP 384 API. Patch by Robin Schreiber. -- cgit v0.12 From 58440c91ce44c9b0f0695c5b62678984f37fa58c Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 14 Jun 2012 21:51:12 +0200 Subject: Issue #15070: fix VS9.0 build regression --- PC/VS9.0/pythoncore.vcproj | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/PC/VS9.0/pythoncore.vcproj b/PC/VS9.0/pythoncore.vcproj index 44a1e5f..9fb63ff 100644 --- a/PC/VS9.0/pythoncore.vcproj +++ b/PC/VS9.0/pythoncore.vcproj @@ -803,6 +803,10 @@ > + + @@ -1563,6 +1567,10 @@ > + + -- cgit v0.12 From fedb04a37aff9f7a2cfe746f7fc4683e74e38bf0 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 14 Jun 2012 21:54:24 +0200 Subject: Update .hgignore for VS9.0-generated files --- .hgignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.hgignore b/.hgignore index 4e4c2ce..5d2b149 100644 --- a/.hgignore +++ b/.hgignore @@ -55,6 +55,8 @@ PC/python_nt*.h PC/pythonnt_rc*.h PC/*.obj PC/*.exe +PC/*/*.exe +PC/*/*.pdb PC/*/*.user PC/*/*.ncb PC/*/*.suo -- cgit v0.12 From 64d11e60f23f6b1435704adb87ebf818e5a4c0c1 Mon Sep 17 00:00:00 2001 From: Eli Bendersky Date: Fri, 15 Jun 2012 07:42:50 +0300 Subject: Replace the iter/itertext methods of Element in _elementtree with true C implementations, instead of the bootstrapped Python code. In addition to being cleaner (removing the last remains of the bootstrapping code in _elementtree), this gives a 10x performance boost for iter() on large documents. Also reorganized the tests a bit to be more robust. --- Lib/test/test_xml_etree.py | 247 ++++++++++++++++------------- Lib/test/test_xml_etree_c.py | 28 +--- Lib/xml/etree/ElementTree.py | 6 +- Modules/_elementtree.c | 362 ++++++++++++++++++++++++++++++++----------- 4 files changed, 414 insertions(+), 229 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 49a5633..bee6329 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -23,7 +23,8 @@ import weakref from test import support from test.support import findfile, import_fresh_module, gc_collect -pyET = import_fresh_module('xml.etree.ElementTree', blocked=['_elementtree']) +pyET = None +ET = None SIMPLE_XMLFILE = findfile("simple.xml", subdir="xmltestdata") try: @@ -209,10 +210,8 @@ def interface(): These methods return an iterable. See bug 6472. - >>> check_method(element.iter("tag").__next__) >>> check_method(element.iterfind("tag").__next__) >>> check_method(element.iterfind("*").__next__) - >>> check_method(tree.iter("tag").__next__) >>> check_method(tree.iterfind("tag").__next__) >>> check_method(tree.iterfind("*").__next__) @@ -291,42 +290,6 @@ def cdata(): 'hello' """ -# Only with Python implementation -def simplefind(): - """ - Test find methods using the elementpath fallback. - - >>> ElementTree = pyET - - >>> CurrentElementPath = ElementTree.ElementPath - >>> ElementTree.ElementPath = ElementTree._SimpleElementPath() - >>> elem = ElementTree.XML(SAMPLE_XML) - >>> elem.find("tag").tag - 'tag' - >>> ElementTree.ElementTree(elem).find("tag").tag - 'tag' - >>> elem.findtext("tag") - 'text' - >>> elem.findtext("tog") - >>> elem.findtext("tog", "default") - 'default' - >>> ElementTree.ElementTree(elem).findtext("tag") - 'text' - >>> summarize_list(elem.findall("tag")) - ['tag', 'tag'] - >>> summarize_list(elem.findall(".//tag")) - ['tag', 'tag', 'tag'] - - Path syntax doesn't work in this case. - - >>> elem.find("section/tag") - >>> elem.findtext("section/tag") - >>> summarize_list(elem.findall("section/tag")) - [] - - >>> ElementTree.ElementPath = CurrentElementPath - """ - def find(): """ Test find methods (including xpath syntax). @@ -1002,36 +965,6 @@ def methods(): '1 < 2\n' """ -def iterators(): - """ - Test iterators. - - >>> e = ET.XML("this is a paragraph...") - >>> summarize_list(e.iter()) - ['html', 'body', 'i'] - >>> summarize_list(e.find("body").iter()) - ['body', 'i'] - >>> summarize(next(e.iter())) - 'html' - >>> "".join(e.itertext()) - 'this is a paragraph...' - >>> "".join(e.find("body").itertext()) - 'this is a paragraph.' - >>> next(e.itertext()) - 'this is a ' - - Method iterparse should return an iterator. See bug 6472. - - >>> sourcefile = serialize(e, to_string=False) - >>> next(ET.iterparse(sourcefile)) # doctest: +ELLIPSIS - ('end', ) - - >>> tree = ET.ElementTree(None) - >>> tree.iter() - Traceback (most recent call last): - AttributeError: 'NoneType' object has no attribute 'iter' - """ - ENTITY_XML = """\ @@ -1339,6 +1272,7 @@ XINCLUDE["default.xml"] = """\ """.format(html.escape(SIMPLE_XMLFILE, True)) + def xinclude_loader(href, parse="xml", encoding=None): try: data = XINCLUDE[href] @@ -1411,22 +1345,6 @@ def xinclude(): >>> # print(serialize(document)) # C5 """ -def xinclude_default(): - """ - >>> from xml.etree import ElementInclude - - >>> document = xinclude_loader("default.xml") - >>> ElementInclude.include(document) - >>> print(serialize(document)) # default - -

Example.

- - text - texttail - - - - """ # # badly formatted xi:include tags @@ -1917,9 +1835,8 @@ class ElementTreeTest(unittest.TestCase): self.assertIsInstance(ET.QName, type) self.assertIsInstance(ET.ElementTree, type) self.assertIsInstance(ET.Element, type) - # XXX issue 14128 with C ElementTree - # self.assertIsInstance(ET.TreeBuilder, type) - # self.assertIsInstance(ET.XMLParser, type) + self.assertIsInstance(ET.TreeBuilder, type) + self.assertIsInstance(ET.XMLParser, type) def test_Element_subclass_trivial(self): class MyElement(ET.Element): @@ -1953,6 +1870,73 @@ class ElementTreeTest(unittest.TestCase): self.assertEqual(mye.newmethod(), 'joe') +class ElementIterTest(unittest.TestCase): + def _ilist(self, elem, tag=None): + return summarize_list(elem.iter(tag)) + + def test_basic(self): + doc = ET.XML("this is a paragraph...") + self.assertEqual(self._ilist(doc), ['html', 'body', 'i']) + self.assertEqual(self._ilist(doc.find('body')), ['body', 'i']) + self.assertEqual(next(doc.iter()).tag, 'html') + self.assertEqual(''.join(doc.itertext()), 'this is a paragraph...') + self.assertEqual(''.join(doc.find('body').itertext()), + 'this is a paragraph.') + self.assertEqual(next(doc.itertext()), 'this is a ') + + # iterparse should return an iterator + sourcefile = serialize(doc, to_string=False) + self.assertEqual(next(ET.iterparse(sourcefile))[0], 'end') + + tree = ET.ElementTree(None) + self.assertRaises(AttributeError, tree.iter) + + def test_corners(self): + # single root, no subelements + a = ET.Element('a') + self.assertEqual(self._ilist(a), ['a']) + + # one child + b = ET.SubElement(a, 'b') + self.assertEqual(self._ilist(a), ['a', 'b']) + + # one child and one grandchild + c = ET.SubElement(b, 'c') + self.assertEqual(self._ilist(a), ['a', 'b', 'c']) + + # two children, only first with grandchild + d = ET.SubElement(a, 'd') + self.assertEqual(self._ilist(a), ['a', 'b', 'c', 'd']) + + # replace first child by second + a[0] = a[1] + del a[1] + self.assertEqual(self._ilist(a), ['a', 'd']) + + def test_iter_by_tag(self): + doc = ET.XML(''' + + + bedroom1 + bedroom2 + + nothing here + + + bedroom8 + + ''') + + self.assertEqual(self._ilist(doc, 'room'), ['room'] * 3) + self.assertEqual(self._ilist(doc, 'house'), ['house'] * 2) + + # make sure both tag=None and tag='*' return all tags + all_tags = ['document', 'house', 'room', 'room', + 'shed', 'house', 'room'] + self.assertEqual(self._ilist(doc), all_tags) + self.assertEqual(self._ilist(doc, '*'), all_tags) + + class TreeBuilderTest(unittest.TestCase): sample1 = (' +

Example.

+ + text + texttail + + +''') class XMLParserTest(unittest.TestCase): sample1 = '22' sample2 = ('>> cElementTree = cET - >>> e = cElementTree.Element('a') - >>> getattr(e, '\uD800') # doctest: +ELLIPSIS - Traceback (most recent call last): - ... - UnicodeEncodeError: ... - - >>> p = cElementTree.XMLParser() - >>> p.version.split()[0] - 'Expat' - >>> getattr(p, '\uD800') - Traceback (most recent call last): - ... - AttributeError: 'XMLParser' object has no attribute '\ud800' - """ - - class MiscTests(unittest.TestCase): # Issue #8651. @support.bigmemtest(size=support._2G + 100, memuse=1) @@ -46,6 +21,7 @@ class MiscTests(unittest.TestCase): finally: data = None + @unittest.skipUnless(cET, 'requires _elementtree') class TestAliasWorking(unittest.TestCase): # Test that the cET alias module is alive @@ -53,6 +29,7 @@ class TestAliasWorking(unittest.TestCase): e = cET_alias.Element('foo') self.assertEqual(e.tag, 'foo') + @unittest.skipUnless(cET, 'requires _elementtree') class TestAcceleratorImported(unittest.TestCase): # Test that the C accelerator was imported, as expected @@ -67,7 +44,6 @@ def test_main(): from test import test_xml_etree, test_xml_etree_c # Run the tests specific to the C implementation - support.run_doctest(test_xml_etree_c, verbosity=True) support.run_unittest( MiscTests, TestAliasWorking, diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index e068fc2..4776625 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -916,11 +916,7 @@ def _namespaces(elem, default_namespace=None): _raise_serialization_error(qname) # populate qname and namespaces table - try: - iterate = elem.iter - except AttributeError: - iterate = elem.getiterator # cET compatibility - for elem in iterate(): + for elem in elem.iter(): tag = elem.tag if isinstance(tag, QName): if tag.text not in qnames: diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 103e778..f0b5a3f 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -103,8 +103,6 @@ do { memory -= size; printf("%8d - %s\n", memory, comment); } while (0) /* glue functions (see the init function for details) */ static PyObject* elementtree_parseerror_obj; static PyObject* elementtree_deepcopy_obj; -static PyObject* elementtree_iter_obj; -static PyObject* elementtree_itertext_obj; static PyObject* elementpath_obj; /* helpers */ @@ -1109,67 +1107,32 @@ element_getchildren(ElementObject* self, PyObject* args) return list; } -static PyObject* -element_iter(ElementObject* self, PyObject* args) -{ - PyObject* result; - PyObject* tag = Py_None; - if (!PyArg_ParseTuple(args, "|O:iter", &tag)) - return NULL; +static PyObject * +create_elementiter(ElementObject *self, PyObject *tag, int gettext); - if (!elementtree_iter_obj) { - PyErr_SetString( - PyExc_RuntimeError, - "iter helper not found" - ); - return NULL; - } - args = PyTuple_New(2); - if (!args) +static PyObject * +element_iter(ElementObject *self, PyObject *args) +{ + PyObject* tag = Py_None; + if (!PyArg_ParseTuple(args, "|O:iter", &tag)) return NULL; - Py_INCREF(self); PyTuple_SET_ITEM(args, 0, (PyObject*) self); - Py_INCREF(tag); PyTuple_SET_ITEM(args, 1, (PyObject*) tag); - - result = PyObject_CallObject(elementtree_iter_obj, args); - - Py_DECREF(args); - - return result; + return create_elementiter(self, tag, 0); } static PyObject* element_itertext(ElementObject* self, PyObject* args) { - PyObject* result; - if (!PyArg_ParseTuple(args, ":itertext")) return NULL; - if (!elementtree_itertext_obj) { - PyErr_SetString( - PyExc_RuntimeError, - "itertext helper not found" - ); - return NULL; - } - - args = PyTuple_New(1); - if (!args) - return NULL; - - Py_INCREF(self); PyTuple_SET_ITEM(args, 0, (PyObject*) self); - - result = PyObject_CallObject(elementtree_itertext_obj, args); - - Py_DECREF(args); - - return result; + return create_elementiter(self, Py_None, 1); } + static PyObject* element_getitem(PyObject* self_, Py_ssize_t index) { @@ -1790,6 +1753,267 @@ static PyTypeObject Element_Type = { 0, /* tp_free */ }; +/******************************* Element iterator ****************************/ + +/* ElementIterObject represents the iteration state over an XML element in + * pre-order traversal. To keep track of which sub-element should be returned + * next, a stack of parents is maintained. This is a standard stack-based + * iterative pre-order traversal of a tree. + * The stack is managed using a single-linked list starting at parent_stack. + * Each stack node contains the saved parent to which we should return after + * the current one is exhausted, and the next child to examine in that parent. + */ +typedef struct ParentLocator_t { + ElementObject *parent; + Py_ssize_t child_index; + struct ParentLocator_t *next; +} ParentLocator; + +typedef struct { + PyObject_HEAD + ParentLocator *parent_stack; + ElementObject *root_element; + PyObject *sought_tag; + int root_done; + int gettext; +} ElementIterObject; + + +static void +elementiter_dealloc(ElementIterObject *it) +{ + ParentLocator *p = it->parent_stack; + while (p) { + ParentLocator *temp = p; + Py_XDECREF(p->parent); + p = p->next; + PyObject_Free(temp); + } + + Py_XDECREF(it->sought_tag); + Py_XDECREF(it->root_element); + + PyObject_GC_UnTrack(it); + PyObject_GC_Del(it); +} + +static int +elementiter_traverse(ElementIterObject *it, visitproc visit, void *arg) +{ + ParentLocator *p = it->parent_stack; + while (p) { + Py_VISIT(p->parent); + p = p->next; + } + + Py_VISIT(it->root_element); + Py_VISIT(it->sought_tag); + return 0; +} + +/* Helper function for elementiter_next. Add a new parent to the parent stack. + */ +static ParentLocator * +parent_stack_push_new(ParentLocator *stack, ElementObject *parent) +{ + ParentLocator *new_node = PyObject_Malloc(sizeof(ParentLocator)); + if (new_node) { + new_node->parent = parent; + Py_INCREF(parent); + new_node->child_index = 0; + new_node->next = stack; + } + return new_node; +} + +static PyObject * +elementiter_next(ElementIterObject *it) +{ + /* Sub-element iterator. + * + * A short note on gettext: this function serves both the iter() and + * itertext() methods to avoid code duplication. However, there are a few + * small differences in the way these iterations work. Namely: + * - itertext() only yields text from nodes that have it, and continues + * iterating when a node doesn't have text (so it doesn't return any + * node like iter()) + * - itertext() also has to handle tail, after finishing with all the + * children of a node. + */ + + while (1) { + /* Handle the case reached in the beginning and end of iteration, where + * the parent stack is empty. The root_done flag gives us indication + * whether we've just started iterating (so root_done is 0), in which + * case the root is returned. If root_done is 1 and we're here, the + * iterator is exhausted. + */ + if (!it->parent_stack->parent) { + if (it->root_done) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } else { + it->parent_stack = parent_stack_push_new(it->parent_stack, + it->root_element); + if (!it->parent_stack) { + PyErr_NoMemory(); + return NULL; + } + + it->root_done = 1; + if (it->sought_tag == Py_None || + PyObject_RichCompareBool(it->root_element->tag, + it->sought_tag, Py_EQ) == 1) { + if (it->gettext) { + PyObject *text = JOIN_OBJ(it->root_element->text); + if (PyObject_IsTrue(text)) { + Py_INCREF(text); + return text; + } + } else { + Py_INCREF(it->root_element); + return (PyObject *)it->root_element; + } + } + } + } + + /* See if there are children left to traverse in the current parent. If + * yes, visit the next child. If not, pop the stack and try again. + */ + ElementObject *cur_parent = it->parent_stack->parent; + Py_ssize_t child_index = it->parent_stack->child_index; + if (cur_parent->extra && child_index < cur_parent->extra->length) { + ElementObject *child = (ElementObject *) + cur_parent->extra->children[child_index]; + it->parent_stack->child_index++; + it->parent_stack = parent_stack_push_new(it->parent_stack, + child); + if (!it->parent_stack) { + PyErr_NoMemory(); + return NULL; + } + + if (it->gettext) { + PyObject *text = JOIN_OBJ(child->text); + if (PyObject_IsTrue(text)) { + Py_INCREF(text); + return text; + } + } else if (it->sought_tag == Py_None || + PyObject_RichCompareBool(child->tag, + it->sought_tag, Py_EQ) == 1) { + Py_INCREF(child); + return (PyObject *)child; + } + else + continue; + } + else { + PyObject *tail = it->gettext ? JOIN_OBJ(cur_parent->tail) : Py_None; + ParentLocator *next = it->parent_stack->next; + Py_XDECREF(it->parent_stack->parent); + PyObject_Free(it->parent_stack); + it->parent_stack = next; + + /* Note that extra condition on it->parent_stack->parent here; + * this is because itertext() is supposed to only return *inner* + * text, not text following the element it began iteration with. + */ + if (it->parent_stack->parent && PyObject_IsTrue(tail)) { + Py_INCREF(tail); + return tail; + } + } + } + + return NULL; +} + + +static PyTypeObject ElementIter_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_elementtree._element_iterator", /* tp_name */ + sizeof(ElementIterObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)elementiter_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + 0, /* tp_doc */ + (traverseproc)elementiter_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + (iternextfunc)elementiter_next, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; + + +static PyObject * +create_elementiter(ElementObject *self, PyObject *tag, int gettext) +{ + ElementIterObject *it; + PyObject *star = NULL; + + it = PyObject_GC_New(ElementIterObject, &ElementIter_Type); + if (!it) + return NULL; + if (!(it->parent_stack = PyObject_Malloc(sizeof(ParentLocator)))) { + PyObject_GC_Del(it); + return NULL; + } + + it->parent_stack->parent = NULL; + it->parent_stack->child_index = 0; + it->parent_stack->next = NULL; + + if (PyUnicode_Check(tag)) + star = PyUnicode_FromString("*"); + else if (PyBytes_Check(tag)) + star = PyBytes_FromString("*"); + + if (star && PyObject_RichCompareBool(tag, star, Py_EQ) == 1) + tag = Py_None; + + Py_XDECREF(star); + it->sought_tag = tag; + it->root_done = 0; + it->gettext = gettext; + it->root_element = self; + + Py_INCREF(self); + Py_INCREF(tag); + + PyObject_GC_Track(it); + return (PyObject *)it; +} + + /* ==================================================================== */ /* the tree builder type */ @@ -3238,8 +3462,7 @@ static struct PyModuleDef _elementtreemodule = { PyMODINIT_FUNC PyInit__elementtree(void) { - PyObject *m, *g, *temp; - char* bootstrap; + PyObject *m, *temp; /* Initialize object types */ if (PyType_Ready(&TreeBuilder_Type) < 0) @@ -3255,44 +3478,6 @@ PyInit__elementtree(void) if (!m) return NULL; - /* The code below requires that the module gets already added - to sys.modules. */ - PyDict_SetItemString(PyImport_GetModuleDict(), - _elementtreemodule.m_name, - m); - - /* python glue code */ - - g = PyDict_New(); - if (!g) - return NULL; - - PyDict_SetItemString(g, "__builtins__", PyEval_GetBuiltins()); - - bootstrap = ( - "def iter(node, tag=None):\n" /* helper */ - " if tag == '*':\n" - " tag = None\n" - " if tag is None or node.tag == tag:\n" - " yield node\n" - " for node in node:\n" - " for node in iter(node, tag):\n" - " yield node\n" - - "def itertext(node):\n" /* helper */ - " if node.text:\n" - " yield node.text\n" - " for e in node:\n" - " for s in e.itertext():\n" - " yield s\n" - " if e.tail:\n" - " yield e.tail\n" - - ); - - if (!PyRun_String(bootstrap, Py_file_input, g, NULL)) - return NULL; - if (!(temp = PyImport_ImportModule("copy"))) return NULL; elementtree_deepcopy_obj = PyObject_GetAttrString(temp, "deepcopy"); @@ -3301,9 +3486,6 @@ PyInit__elementtree(void) if (!(elementpath_obj = PyImport_ImportModule("xml.etree.ElementPath"))) return NULL; - elementtree_iter_obj = PyDict_GetItemString(g, "iter"); - elementtree_itertext_obj = PyDict_GetItemString(g, "itertext"); - /* link against pyexpat */ expat_capi = PyCapsule_Import(PyExpat_CAPSULE_NAME, 0); if (expat_capi) { -- cgit v0.12 From 113da642591d9e290c7d24a7e5c68c0f040ce42f Mon Sep 17 00:00:00 2001 From: Eli Bendersky Date: Fri, 15 Jun 2012 07:52:49 +0300 Subject: Fix windows compilation problems caused by previous commit. --- Modules/_elementtree.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index f0b5a3f..cb84048 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -1840,6 +1840,8 @@ elementiter_next(ElementIterObject *it) * - itertext() also has to handle tail, after finishing with all the * children of a node. */ + ElementObject *cur_parent; + Py_ssize_t child_index; while (1) { /* Handle the case reached in the beginning and end of iteration, where @@ -1881,8 +1883,8 @@ elementiter_next(ElementIterObject *it) /* See if there are children left to traverse in the current parent. If * yes, visit the next child. If not, pop the stack and try again. */ - ElementObject *cur_parent = it->parent_stack->parent; - Py_ssize_t child_index = it->parent_stack->child_index; + cur_parent = it->parent_stack->parent; + child_index = it->parent_stack->child_index; if (cur_parent->extra && child_index < cur_parent->extra->length) { ElementObject *child = (ElementObject *) cur_parent->extra->children[child_index]; -- cgit v0.12 From 175fada42943d627dea4ebd9527ca1d6e9b62cca Mon Sep 17 00:00:00 2001 From: Eli Bendersky Date: Fri, 15 Jun 2012 08:37:08 +0300 Subject: mark problematic test as expected failure - investigating --- Lib/test/test_xml_etree.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index bee6329..22a1773 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -2010,7 +2010,9 @@ class TreeBuilderTest(unittest.TestCase): ('html', '-//W3C//DTD XHTML 1.0 Transitional//EN', 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd')) + class XincludeTest(unittest.TestCase): + @unittest.expectedFailure def test_xinclude_default(self): from xml.etree import ElementInclude doc = xinclude_loader('default.xml') @@ -2024,6 +2026,8 @@ class XincludeTest(unittest.TestCase): ''') + + class XMLParserTest(unittest.TestCase): sample1 = '22' sample2 = (' Date: Fri, 15 Jun 2012 09:03:19 +0300 Subject: Removed _SimpleElementPath and its flaky test. The test monkey-patches the module, which causes other failures and fails itself depending on the order tests are run. --- Lib/test/test_xml_etree.py | 23 ----------------------- Lib/xml/etree/ElementTree.py | 26 +------------------------- 2 files changed, 1 insertion(+), 48 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 22a1773..87398b4 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -2234,28 +2234,6 @@ class NoAcceleratorTest(unittest.TestCase): self.assertEqual(pyET.Element.__module__, 'xml.etree.ElementTree') self.assertEqual(pyET.SubElement.__module__, 'xml.etree.ElementTree') - -class ElementPathFallbackTest(unittest.TestCase): - def test_fallback(self): - current_ElementPath = ET.ElementPath - ET.ElementPath = ET._SimpleElementPath() - elem = ET.XML(SAMPLE_XML) - self.assertEqual(elem.find('tag').tag, 'tag') - self.assertEqual(ET.ElementTree(elem).find('tag').tag, 'tag') - self.assertEqual(elem.findtext('tag'), 'text') - self.assertIsNone(elem.findtext('tog')) - self.assertEqual(elem.findtext('tog', 'default'), 'default') - self.assertEqual(ET.ElementTree(elem).findtext('tag'), 'text') - self.assertEqual(summarize_list(elem.findall('tag')), ['tag', 'tag']) - self.assertEqual(summarize_list(elem.findall('.//tag')), - ['tag', 'tag', 'tag']) - - #self.assertIsNone(elem.find('section/tag')) - #self.assertIsNone(elem.findtext('section/tag')) - self.assertEqual(summarize_list(elem.findall('section/tag')), []) - - ET.ElementPath = current_ElementPath - # -------------------------------------------------------------------- @@ -2328,7 +2306,6 @@ def test_main(module=None): if pyET: test_classes.extend([ NoAcceleratorTest, - ElementPathFallbackTest, ]) support.run_unittest(*test_classes) diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 4776625..d30a83c 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -101,32 +101,8 @@ import sys import re import warnings -class _SimpleElementPath: - # emulate pre-1.2 find/findtext/findall behaviour - def find(self, element, tag, namespaces=None): - for elem in element: - if elem.tag == tag: - return elem - return None - def findtext(self, element, tag, default=None, namespaces=None): - elem = self.find(element, tag) - if elem is None: - return default - return elem.text or "" - def iterfind(self, element, tag, namespaces=None): - if tag[:3] == ".//": - for elem in element.iter(tag[3:]): - yield elem - for elem in element: - if elem.tag == tag: - yield elem - def findall(self, element, tag, namespaces=None): - return list(self.iterfind(element, tag, namespaces)) +from . import ElementPath -try: - from . import ElementPath -except ImportError: - ElementPath = _SimpleElementPath() ## # Parser error. This is a subclass of SyntaxError. -- cgit v0.12 From 307693a8bb3b5b5ff2e3a0d5a0a837289e4de8be Mon Sep 17 00:00:00 2001 From: Eli Bendersky Date: Fri, 15 Jun 2012 09:40:44 +0300 Subject: Skip XincludeTest entirely instead of just ignoring failures, because it may segfault, depending on the order of running tests --- Lib/test/test_xml_etree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 87398b4..24ecae5 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -2011,8 +2011,8 @@ class TreeBuilderTest(unittest.TestCase): 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd')) +@unittest.skip('Unstable due to module monkeypatching') class XincludeTest(unittest.TestCase): - @unittest.expectedFailure def test_xinclude_default(self): from xml.etree import ElementInclude doc = xinclude_loader('default.xml') -- cgit v0.12 From 807770ec1bead8aff0716384621638ed80e9f56b Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Fri, 15 Jun 2012 21:14:08 +1000 Subject: Issue #15061: Don't oversell the capabilities of the new non-shortcircuiting comparison function in hmac --- Doc/library/hmac.rst | 41 +++++++++++++++++++++++++---------------- Lib/hmac.py | 28 ++++++++++++++-------------- Lib/test/test_hmac.py | 44 +++++++++++++++++++++++--------------------- Misc/NEWS | 5 +++++ 4 files changed, 67 insertions(+), 51 deletions(-) diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index 8274bb1..e6ce99b 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -42,8 +42,8 @@ An HMAC object has the following methods: When comparing the output of :meth:`digest` to an externally-supplied digest during a verification routine, it is recommended to use the - :func:`hmac.secure_compare` function instead of the ``==`` operator - to avoid potential timing attacks. + :func:`compare_digest` function instead of the ``==`` operator + to reduce the vulnerability to timing attacks. .. method:: HMAC.hexdigest() @@ -54,10 +54,11 @@ An HMAC object has the following methods: .. warning:: - When comparing the output of :meth:`hexdigest` to an externally-supplied - digest during a verification routine, it is recommended to use the - :func:`hmac.secure_compare` function instead of the ``==`` operator - to avoid potential timing attacks. + The output of :meth:`hexdigest` should not be compared directly to an + externally-supplied digest during a verification routine. Instead, the + externally supplied digest should be converted to a :class:`bytes` + value and compared to the output of :meth:`digest` with + :func:`compare_digest`. .. method:: HMAC.copy() @@ -68,20 +69,28 @@ An HMAC object has the following methods: This module also provides the following helper function: -.. function:: secure_compare(a, b) +.. function:: compare_digest(a, b) + + Returns the equivalent of ``a == b``, but avoids content based + short circuiting behaviour to reduce the vulnerability to timing + analysis. The inputs must be :class:`bytes` instances. - Returns the equivalent of ``a == b``, but using a time-independent - comparison method. Comparing the full lengths of the inputs *a* and *b*, - instead of short-circuiting the comparison upon the first unequal byte, - prevents leaking information about the inputs being compared and mitigates - potential timing attacks. The inputs must be either :class:`str` or - :class:`bytes` instances. + Using a short circuiting comparison (that is, one that terminates as soon + as it finds any difference between the values) to check digests for + correctness can be problematic, as it introduces a potential + vulnerability when an attacker can control both the message to be checked + *and* the purported signature value. By keeping the plaintext consistent + and supplying different signature values, an attacker may be able to use + timing variations to search the signature space for the expected value in + O(n) time rather than the desired O(2**n). .. note:: - While the :func:`hmac.secure_compare` function prevents leaking the - contents of the inputs via a timing attack, it does leak the length - of the inputs. However, this generally is not a security risk. + While this function reduces the likelihood of leaking the contents of + the expected digest via a timing attack, it still uses short circuiting + behaviour based on the *length* of the inputs. It is assumed that the + expected length of the digest is not a secret, as it is typically + published as part of a file format, network protocol or API definition. .. versionadded:: 3.3 diff --git a/Lib/hmac.py b/Lib/hmac.py index 13ffdbe..e47965b 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -13,24 +13,24 @@ trans_36 = bytes((x ^ 0x36) for x in range(256)) digest_size = None -def secure_compare(a, b): - """Returns the equivalent of 'a == b', but using a time-independent - comparison method to prevent timing attacks.""" - if not ((isinstance(a, str) and isinstance(b, str)) or - (isinstance(a, bytes) and isinstance(b, bytes))): - raise TypeError("inputs must be strings or bytes") - +def compare_digest(a, b): + """Returns the equivalent of 'a == b', but avoids content based short + circuiting to reduce the vulnerability to timing attacks.""" + # Consistent timing matters more here than data type flexibility + if not (isinstance(a, bytes) and isinstance(b, bytes)): + raise TypeError("inputs must be bytes instances") + + # We assume the length of the expected digest is public knowledge, + # thus this early return isn't leaking anything an attacker wouldn't + # already know if len(a) != len(b): return False + # We assume that integers in the bytes range are all cached, + # thus timing shouldn't vary much due to integer object creation result = 0 - if isinstance(a, bytes): - for x, y in zip(a, b): - result |= x ^ y - else: - for x, y in zip(a, b): - result |= ord(x) ^ ord(y) - + for x, y in zip(a, b): + result |= x ^ y return result == 0 diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index 042bc5d..4e5961d 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -302,40 +302,42 @@ class CopyTestCase(unittest.TestCase): self.assertEqual(h1.hexdigest(), h2.hexdigest(), "Hexdigest of copy doesn't match original hexdigest.") -class SecureCompareTestCase(unittest.TestCase): +class CompareDigestTestCase(unittest.TestCase): def test_compare(self): # Testing input type exception handling a, b = 100, 200 - self.assertRaises(TypeError, hmac.secure_compare, a, b) - a, b = 100, "foobar" - self.assertRaises(TypeError, hmac.secure_compare, a, b) + self.assertRaises(TypeError, hmac.compare_digest, a, b) + a, b = 100, b"foobar" + self.assertRaises(TypeError, hmac.compare_digest, a, b) + a, b = b"foobar", 200 + self.assertRaises(TypeError, hmac.compare_digest, a, b) a, b = "foobar", b"foobar" - self.assertRaises(TypeError, hmac.secure_compare, a, b) + self.assertRaises(TypeError, hmac.compare_digest, a, b) + a, b = b"foobar", "foobar" + self.assertRaises(TypeError, hmac.compare_digest, a, b) + a, b = "foobar", "foobar" + self.assertRaises(TypeError, hmac.compare_digest, a, b) + a, b = bytearray(b"foobar"), bytearray(b"foobar") + self.assertRaises(TypeError, hmac.compare_digest, a, b) - # Testing str/bytes of different lengths - a, b = "foobar", "foo" - self.assertFalse(hmac.secure_compare(a, b)) + # Testing bytes of different lengths a, b = b"foobar", b"foo" - self.assertFalse(hmac.secure_compare(a, b)) + self.assertFalse(hmac.compare_digest(a, b)) a, b = b"\xde\xad\xbe\xef", b"\xde\xad" - self.assertFalse(hmac.secure_compare(a, b)) + self.assertFalse(hmac.compare_digest(a, b)) - # Testing str/bytes of same lengths, different values - a, b = "foobar", "foobaz" - self.assertFalse(hmac.secure_compare(a, b)) + # Testing bytes of same lengths, different values a, b = b"foobar", b"foobaz" - self.assertFalse(hmac.secure_compare(a, b)) + self.assertFalse(hmac.compare_digest(a, b)) a, b = b"\xde\xad\xbe\xef", b"\xab\xad\x1d\xea" - self.assertFalse(hmac.secure_compare(a, b)) + self.assertFalse(hmac.compare_digest(a, b)) - # Testing str/bytes of same lengths, same values - a, b = "foobar", "foobar" - self.assertTrue(hmac.secure_compare(a, b)) + # Testing bytes of same lengths, same values a, b = b"foobar", b"foobar" - self.assertTrue(hmac.secure_compare(a, b)) + self.assertTrue(hmac.compare_digest(a, b)) a, b = b"\xde\xad\xbe\xef", b"\xde\xad\xbe\xef" - self.assertTrue(hmac.secure_compare(a, b)) + self.assertTrue(hmac.compare_digest(a, b)) def test_main(): support.run_unittest( @@ -343,7 +345,7 @@ def test_main(): ConstructorTestCase, SanityTestCase, CopyTestCase, - SecureCompareTestCase + CompareDigestTestCase ) if __name__ == "__main__": diff --git a/Misc/NEWS b/Misc/NEWS index 058555f..7d7d434 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -21,6 +21,11 @@ Core and Builtins Library ------- +- Issue #15061: The inappropriately named hmac.secure_compare has been + renamed to hash.compare_digest, restricted to operating on bytes inputs + only and had its documentation updated to more acurrately reflect both its + intent and its limitations + - Issue #13841: Make child processes exit using sys.exit() on Windows. - Issue #14936: curses_panel was converted to PEP 3121 and PEP 384 API. -- cgit v0.12 From 96f806bccbab50aa9af247cffd6a07d1c225827f Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Fri, 15 Jun 2012 21:16:25 +1000 Subject: Fix typos in NEWS entry --- Misc/NEWS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 7d7d434..b02e1f2 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -22,8 +22,8 @@ Library ------- - Issue #15061: The inappropriately named hmac.secure_compare has been - renamed to hash.compare_digest, restricted to operating on bytes inputs - only and had its documentation updated to more acurrately reflect both its + renamed to hmac.compare_digest, restricted to operating on bytes inputs + only and had its documentation updated to more accurately reflect both its intent and its limitations - Issue #13841: Make child processes exit using sys.exit() on Windows. -- cgit v0.12 From 13775221e06f86b319f9591946b7ff9fb49bb907 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 15 Jun 2012 19:11:31 +0200 Subject: Issue #14933: fix misleading doc about weakref support in extension types. --- Doc/extending/newtypes.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 2ba01bc..8f18253 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -1459,9 +1459,8 @@ The type constructor is responsible for initializing the weak reference list to } The only further addition is that the destructor needs to call the weak -reference manager to clear any weak references. This should be done before any -other parts of the destruction have occurred, but is only required if the weak -reference list is non-*NULL*:: +reference manager to clear any weak references. This is only required if the +weak reference list is non-*NULL*:: static void instance_dealloc(PyInstanceObject *inst) -- cgit v0.12 From 3730a17a58d9058b0880202050a8c1e4ea193e8f Mon Sep 17 00:00:00 2001 From: Richard Oudkerk Date: Fri, 15 Jun 2012 18:26:07 +0100 Subject: Issue #14059: Implement multiprocessing.Barrier --- Doc/library/multiprocessing.rst | 23 ++- Lib/multiprocessing/__init__.py | 11 +- Lib/multiprocessing/dummy/__init__.py | 4 +- Lib/multiprocessing/managers.py | 21 +++ Lib/multiprocessing/synchronize.py | 40 ++++ Lib/test/test_multiprocessing.py | 337 +++++++++++++++++++++++++++++++++- Misc/NEWS | 2 + 7 files changed, 426 insertions(+), 12 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 5e56ecb..4171977 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -226,11 +226,11 @@ However, if you really do need to use some shared data then holds Python objects and allows other processes to manipulate them using proxies. - A manager returned by :func:`Manager` will support types :class:`list`, - :class:`dict`, :class:`Namespace`, :class:`Lock`, :class:`RLock`, - :class:`Semaphore`, :class:`BoundedSemaphore`, :class:`Condition`, - :class:`Event`, :class:`Queue`, :class:`Value` and :class:`Array`. For - example, :: + A manager returned by :func:`Manager` will support types + :class:`list`, :class:`dict`, :class:`Namespace`, :class:`Lock`, + :class:`RLock`, :class:`Semaphore`, :class:`BoundedSemaphore`, + :class:`Condition`, :class:`Event`, :class:`Barrier`, + :class:`Queue`, :class:`Value` and :class:`Array`. For example, :: from multiprocessing import Process, Manager @@ -885,6 +885,12 @@ program as they are in a multithreaded program. See the documentation for Note that one can also create synchronization primitives by using a manager object -- see :ref:`multiprocessing-managers`. +.. class:: Barrier(parties[, action[, timeout]]) + + A barrier object: a clone of :class:`threading.Barrier`. + + .. versionadded:: 3.3 + .. class:: BoundedSemaphore([value]) A bounded semaphore object: a clone of :class:`threading.BoundedSemaphore`. @@ -1280,6 +1286,13 @@ their parent process exits. The manager classes are defined in the It also supports creation of shared lists and dictionaries. + .. method:: Barrier(parties[, action[, timeout]]) + + Create a shared :class:`threading.Barrier` object and return a + proxy for it. + + .. versionadded:: 3.3 + .. method:: BoundedSemaphore([value]) Create a shared :class:`threading.BoundedSemaphore` object and return a diff --git a/Lib/multiprocessing/__init__.py b/Lib/multiprocessing/__init__.py index 02460f0..1f3e67c 100644 --- a/Lib/multiprocessing/__init__.py +++ b/Lib/multiprocessing/__init__.py @@ -23,8 +23,8 @@ __all__ = [ 'Manager', 'Pipe', 'cpu_count', 'log_to_stderr', 'get_logger', 'allow_connection_pickling', 'BufferTooShort', 'TimeoutError', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', - 'Event', 'Queue', 'SimpleQueue', 'JoinableQueue', 'Pool', 'Value', 'Array', - 'RawValue', 'RawArray', 'SUBDEBUG', 'SUBWARNING', + 'Event', 'Barrier', 'Queue', 'SimpleQueue', 'JoinableQueue', 'Pool', + 'Value', 'Array', 'RawValue', 'RawArray', 'SUBDEBUG', 'SUBWARNING', ] __author__ = 'R. Oudkerk (r.m.oudkerk@gmail.com)' @@ -186,6 +186,13 @@ def Event(): from multiprocessing.synchronize import Event return Event() +def Barrier(parties, action=None, timeout=None): + ''' + Returns a barrier object + ''' + from multiprocessing.synchronize import Barrier + return Barrier(parties, action, timeout) + def Queue(maxsize=0): ''' Returns a queue object diff --git a/Lib/multiprocessing/dummy/__init__.py b/Lib/multiprocessing/dummy/__init__.py index 9bf8f6b..e31fc61 100644 --- a/Lib/multiprocessing/dummy/__init__.py +++ b/Lib/multiprocessing/dummy/__init__.py @@ -35,7 +35,7 @@ __all__ = [ 'Process', 'current_process', 'active_children', 'freeze_support', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', - 'Event', 'Queue', 'Manager', 'Pipe', 'Pool', 'JoinableQueue' + 'Event', 'Barrier', 'Queue', 'Manager', 'Pipe', 'Pool', 'JoinableQueue' ] # @@ -49,7 +49,7 @@ import array from multiprocessing.dummy.connection import Pipe from threading import Lock, RLock, Semaphore, BoundedSemaphore -from threading import Event, Condition +from threading import Event, Condition, Barrier from queue import Queue # diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 817d232..cded4f3 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -993,6 +993,26 @@ class EventProxy(BaseProxy): def wait(self, timeout=None): return self._callmethod('wait', (timeout,)) + +class BarrierProxy(BaseProxy): + _exposed_ = ('__getattribute__', 'wait', 'abort', 'reset') + def wait(self, timeout=None): + return self._callmethod('wait', (timeout,)) + def abort(self): + return self._callmethod('abort') + def reset(self): + return self._callmethod('reset') + @property + def parties(self): + return self._callmethod('__getattribute__', ('parties',)) + @property + def n_waiting(self): + return self._callmethod('__getattribute__', ('n_waiting',)) + @property + def broken(self): + return self._callmethod('__getattribute__', ('broken',)) + + class NamespaceProxy(BaseProxy): _exposed_ = ('__getattribute__', '__setattr__', '__delattr__') def __getattr__(self, key): @@ -1084,6 +1104,7 @@ SyncManager.register('Semaphore', threading.Semaphore, AcquirerProxy) SyncManager.register('BoundedSemaphore', threading.BoundedSemaphore, AcquirerProxy) SyncManager.register('Condition', threading.Condition, ConditionProxy) +SyncManager.register('Barrier', threading.Barrier, BarrierProxy) SyncManager.register('Pool', Pool, PoolProxy) SyncManager.register('list', list, ListProxy) SyncManager.register('dict', dict, DictProxy) diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index 4502a97..22eabe5 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -333,3 +333,43 @@ class Event(object): return False finally: self._cond.release() + +# +# Barrier +# + +class Barrier(threading.Barrier): + + def __init__(self, parties, action=None, timeout=None): + import struct + from multiprocessing.heap import BufferWrapper + wrapper = BufferWrapper(struct.calcsize('i') * 2) + cond = Condition() + self.__setstate__((parties, action, timeout, cond, wrapper)) + self._state = 0 + self._count = 0 + + def __setstate__(self, state): + (self._parties, self._action, self._timeout, + self._cond, self._wrapper) = state + self._array = self._wrapper.create_memoryview().cast('i') + + def __getstate__(self): + return (self._parties, self._action, self._timeout, + self._cond, self._wrapper) + + @property + def _state(self): + return self._array[0] + + @_state.setter + def _state(self, value): + self._array[0] = value + + @property + def _count(self): + return self._array[1] + + @_count.setter + def _count(self, value): + self._array[1] = value diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py index e4ad904..6da5574 100644 --- a/Lib/test/test_multiprocessing.py +++ b/Lib/test/test_multiprocessing.py @@ -18,6 +18,7 @@ import array import socket import random import logging +import struct import test.support @@ -1057,6 +1058,336 @@ class _TestEvent(BaseTestCase): self.assertEqual(wait(), True) # +# Tests for Barrier - adapted from tests in test/lock_tests.py +# + +# Many of the tests for threading.Barrier use a list as an atomic +# counter: a value is appended to increment the counter, and the +# length of the list gives the value. We use the class DummyList +# for the same purpose. + +class _DummyList(object): + + def __init__(self): + wrapper = multiprocessing.heap.BufferWrapper(struct.calcsize('i')) + lock = multiprocessing.Lock() + self.__setstate__((wrapper, lock)) + self._lengthbuf[0] = 0 + + def __setstate__(self, state): + (self._wrapper, self._lock) = state + self._lengthbuf = self._wrapper.create_memoryview().cast('i') + + def __getstate__(self): + return (self._wrapper, self._lock) + + def append(self, _): + with self._lock: + self._lengthbuf[0] += 1 + + def __len__(self): + with self._lock: + return self._lengthbuf[0] + +def _wait(): + # A crude wait/yield function not relying on synchronization primitives. + time.sleep(0.01) + + +class Bunch(object): + """ + A bunch of threads. + """ + def __init__(self, namespace, f, args, n, wait_before_exit=False): + """ + Construct a bunch of `n` threads running the same function `f`. + If `wait_before_exit` is True, the threads won't terminate until + do_finish() is called. + """ + self.f = f + self.args = args + self.n = n + self.started = namespace.DummyList() + self.finished = namespace.DummyList() + self._can_exit = namespace.Value('i', not wait_before_exit) + for i in range(n): + namespace.Process(target=self.task).start() + + def task(self): + pid = os.getpid() + self.started.append(pid) + try: + self.f(*self.args) + finally: + self.finished.append(pid) + while not self._can_exit.value: + _wait() + + def wait_for_started(self): + while len(self.started) < self.n: + _wait() + + def wait_for_finished(self): + while len(self.finished) < self.n: + _wait() + + def do_finish(self): + self._can_exit.value = True + + +class AppendTrue(object): + def __init__(self, obj): + self.obj = obj + def __call__(self): + self.obj.append(True) + + +class _TestBarrier(BaseTestCase): + """ + Tests for Barrier objects. + """ + N = 5 + defaultTimeout = 10.0 # XXX Slow Windows buildbots need generous timeout + + def setUp(self): + self.barrier = self.Barrier(self.N, timeout=self.defaultTimeout) + + def tearDown(self): + self.barrier.abort() + self.barrier = None + + def DummyList(self): + if self.TYPE == 'threads': + return [] + elif self.TYPE == 'manager': + return self.manager.list() + else: + return _DummyList() + + def run_threads(self, f, args): + b = Bunch(self, f, args, self.N-1) + f(*args) + b.wait_for_finished() + + @classmethod + def multipass(cls, barrier, results, n): + m = barrier.parties + assert m == cls.N + for i in range(n): + results[0].append(True) + assert len(results[1]) == i * m + barrier.wait() + results[1].append(True) + assert len(results[0]) == (i + 1) * m + barrier.wait() + try: + assert barrier.n_waiting == 0 + except NotImplementedError: + pass + assert not barrier.broken + + def test_barrier(self, passes=1): + """ + Test that a barrier is passed in lockstep + """ + results = [self.DummyList(), self.DummyList()] + self.run_threads(self.multipass, (self.barrier, results, passes)) + + def test_barrier_10(self): + """ + Test that a barrier works for 10 consecutive runs + """ + return self.test_barrier(10) + + @classmethod + def _test_wait_return_f(cls, barrier, queue): + res = barrier.wait() + queue.put(res) + + def test_wait_return(self): + """ + test the return value from barrier.wait + """ + queue = self.Queue() + self.run_threads(self._test_wait_return_f, (self.barrier, queue)) + results = [queue.get() for i in range(self.N)] + self.assertEqual(results.count(0), 1) + + @classmethod + def _test_action_f(cls, barrier, results): + barrier.wait() + if len(results) != 1: + raise RuntimeError + + def test_action(self): + """ + Test the 'action' callback + """ + results = self.DummyList() + barrier = self.Barrier(self.N, action=AppendTrue(results)) + self.run_threads(self._test_action_f, (barrier, results)) + self.assertEqual(len(results), 1) + + @classmethod + def _test_abort_f(cls, barrier, results1, results2): + try: + i = barrier.wait() + if i == cls.N//2: + raise RuntimeError + barrier.wait() + results1.append(True) + except threading.BrokenBarrierError: + results2.append(True) + except RuntimeError: + barrier.abort() + + def test_abort(self): + """ + Test that an abort will put the barrier in a broken state + """ + results1 = self.DummyList() + results2 = self.DummyList() + self.run_threads(self._test_abort_f, + (self.barrier, results1, results2)) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertTrue(self.barrier.broken) + + @classmethod + def _test_reset_f(cls, barrier, results1, results2, results3): + i = barrier.wait() + if i == cls.N//2: + # Wait until the other threads are all in the barrier. + while barrier.n_waiting < cls.N-1: + time.sleep(0.001) + barrier.reset() + else: + try: + barrier.wait() + results1.append(True) + except threading.BrokenBarrierError: + results2.append(True) + # Now, pass the barrier again + barrier.wait() + results3.append(True) + + def test_reset(self): + """ + Test that a 'reset' on a barrier frees the waiting threads + """ + results1 = self.DummyList() + results2 = self.DummyList() + results3 = self.DummyList() + self.run_threads(self._test_reset_f, + (self.barrier, results1, results2, results3)) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertEqual(len(results3), self.N) + + @classmethod + def _test_abort_and_reset_f(cls, barrier, barrier2, + results1, results2, results3): + try: + i = barrier.wait() + if i == cls.N//2: + raise RuntimeError + barrier.wait() + results1.append(True) + except threading.BrokenBarrierError: + results2.append(True) + except RuntimeError: + barrier.abort() + # Synchronize and reset the barrier. Must synchronize first so + # that everyone has left it when we reset, and after so that no + # one enters it before the reset. + if barrier2.wait() == cls.N//2: + barrier.reset() + barrier2.wait() + barrier.wait() + results3.append(True) + + def test_abort_and_reset(self): + """ + Test that a barrier can be reset after being broken. + """ + results1 = self.DummyList() + results2 = self.DummyList() + results3 = self.DummyList() + barrier2 = self.Barrier(self.N) + + self.run_threads(self._test_abort_and_reset_f, + (self.barrier, barrier2, results1, results2, results3)) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertEqual(len(results3), self.N) + + @classmethod + def _test_timeout_f(cls, barrier, results): + i = barrier.wait(20) + if i == cls.N//2: + # One thread is late! + time.sleep(4.0) + try: + barrier.wait(0.5) + except threading.BrokenBarrierError: + results.append(True) + + def test_timeout(self): + """ + Test wait(timeout) + """ + results = self.DummyList() + self.run_threads(self._test_timeout_f, (self.barrier, results)) + self.assertEqual(len(results), self.barrier.parties) + + @classmethod + def _test_default_timeout_f(cls, barrier, results): + i = barrier.wait(20) + if i == cls.N//2: + # One thread is later than the default timeout + time.sleep(4.0) + try: + barrier.wait() + except threading.BrokenBarrierError: + results.append(True) + + def test_default_timeout(self): + """ + Test the barrier's default timeout + """ + barrier = self.Barrier(self.N, timeout=1.0) + results = self.DummyList() + self.run_threads(self._test_default_timeout_f, (barrier, results)) + self.assertEqual(len(results), barrier.parties) + + def test_single_thread(self): + b = self.Barrier(1) + b.wait() + b.wait() + + @classmethod + def _test_thousand_f(cls, barrier, passes, conn, lock): + for i in range(passes): + barrier.wait() + with lock: + conn.send(i) + + def test_thousand(self): + if self.TYPE == 'manager': + return + passes = 1000 + lock = self.Lock() + conn, child_conn = self.Pipe(False) + for j in range(self.N): + p = self.Process(target=self._test_thousand_f, + args=(self.barrier, passes, child_conn, lock)) + p.start() + + for i in range(passes): + for j in range(self.N): + self.assertEqual(conn.recv(), i) + +# # # @@ -2532,7 +2863,7 @@ class ProcessesMixin(object): Process = multiprocessing.Process locals().update(get_attributes(multiprocessing, ( 'Queue', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', - 'Condition', 'Event', 'Value', 'Array', 'RawValue', + 'Condition', 'Event', 'Barrier', 'Value', 'Array', 'RawValue', 'RawArray', 'current_process', 'active_children', 'Pipe', 'connection', 'JoinableQueue', 'Pool' ))) @@ -2547,7 +2878,7 @@ class ManagerMixin(object): manager = object.__new__(multiprocessing.managers.SyncManager) locals().update(get_attributes(manager, ( 'Queue', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', - 'Condition', 'Event', 'Value', 'Array', 'list', 'dict', + 'Condition', 'Event', 'Barrier', 'Value', 'Array', 'list', 'dict', 'Namespace', 'JoinableQueue', 'Pool' ))) @@ -2560,7 +2891,7 @@ class ThreadsMixin(object): Process = multiprocessing.dummy.Process locals().update(get_attributes(multiprocessing.dummy, ( 'Queue', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', - 'Condition', 'Event', 'Value', 'Array', 'current_process', + 'Condition', 'Event', 'Barrier', 'Value', 'Array', 'current_process', 'active_children', 'Pipe', 'connection', 'dict', 'list', 'Namespace', 'JoinableQueue', 'Pool' ))) diff --git a/Misc/NEWS b/Misc/NEWS index b02e1f2..ef0145d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -21,6 +21,8 @@ Core and Builtins Library ------- +- Issue #14059: Implement multiprocessing.Barrier. + - Issue #15061: The inappropriately named hmac.secure_compare has been renamed to hmac.compare_digest, restricted to operating on bytes inputs only and had its documentation updated to more accurately reflect both its -- cgit v0.12 From 02653f1b111c8b9f64de8960d9a55d4f5821e257 Mon Sep 17 00:00:00 2001 From: Petri Lehtinen Date: Fri, 15 Jun 2012 20:50:51 +0300 Subject: #15036: Make a repeated changes and flushes work with single-file mailboxes --- Lib/mailbox.py | 1 + Lib/test/test_mailbox.py | 11 +++++++++++ Misc/NEWS | 4 ++++ 3 files changed, 16 insertions(+) diff --git a/Lib/mailbox.py b/Lib/mailbox.py index a677729..2be4c83 100644 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -675,6 +675,7 @@ class _singlefileMailbox(Mailbox): new_file.write(buffer) new_toc[key] = (new_start, new_file.tell()) self._post_message_hook(new_file) + self._file_length = new_file.tell() except: new_file.close() os.remove(new_file.name) diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index f885d33e..91c8983 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -500,6 +500,17 @@ class TestMailbox(TestBase): # Write changes to disk self._test_flush_or_close(self._box.flush, True) + def test_popitem_and_flush_twice(self): + # See #15036. + self._box.add(self._template % 0) + self._box.add(self._template % 1) + self._box.flush() + + self._box.popitem() + self._box.flush() + self._box.popitem() + self._box.flush() + def test_lock_unlock(self): # Lock and unlock the mailbox self.assertFalse(os.path.exists(self._get_lock_path())) diff --git a/Misc/NEWS b/Misc/NEWS index 22cf31c..f9abb66 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -70,6 +70,10 @@ Core and Builtins Library ------- +- Issue #15036: Allow removing or changing multiple items in + single-file mailboxes (mbox, MMDF, Babyl) flushing the mailbox + between the changes. + - Issue #10133: Make multiprocessing deallocate buffer if socket read fails. Patch by Hallvard B Furuseth. -- cgit v0.12 From 0f52346e760648a83504b1babd2a530f802eb096 Mon Sep 17 00:00:00 2001 From: Richard Oudkerk Date: Fri, 15 Jun 2012 19:18:30 +0100 Subject: Fix for 2d2f206d040e so that test_multiprocessing does not depend on ctypes --- Lib/test/test_multiprocessing.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py index 6da5574..e4031f6 100644 --- a/Lib/test/test_multiprocessing.py +++ b/Lib/test/test_multiprocessing.py @@ -1109,9 +1109,13 @@ class Bunch(object): self.n = n self.started = namespace.DummyList() self.finished = namespace.DummyList() - self._can_exit = namespace.Value('i', not wait_before_exit) + self._can_exit = namespace.Event() + if not wait_before_exit: + self._can_exit.set() for i in range(n): - namespace.Process(target=self.task).start() + p = namespace.Process(target=self.task) + p.daemon = True + p.start() def task(self): pid = os.getpid() @@ -1120,8 +1124,8 @@ class Bunch(object): self.f(*self.args) finally: self.finished.append(pid) - while not self._can_exit.value: - _wait() + self._can_exit.wait(30) + assert self._can_exit.is_set() def wait_for_started(self): while len(self.started) < self.n: @@ -1132,7 +1136,7 @@ class Bunch(object): _wait() def do_finish(self): - self._can_exit.value = True + self._can_exit.set() class AppendTrue(object): -- cgit v0.12 From 3049f1243ec85590f64962994f055da66c85a15e Mon Sep 17 00:00:00 2001 From: Richard Oudkerk Date: Fri, 15 Jun 2012 20:08:29 +0100 Subject: Increase timeout used when waiting for manager to shutdown cleanly before resorting to terminate() --- Lib/multiprocessing/managers.py | 2 +- Lib/test/test_multiprocessing.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index cded4f3..f6611af 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -582,7 +582,7 @@ class BaseManager(object): except Exception: pass - process.join(timeout=0.2) + process.join(timeout=1.0) if process.is_alive(): util.info('manager still alive') if hasattr(process, 'terminate'): diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py index e4031f6..d79110a 100644 --- a/Lib/test/test_multiprocessing.py +++ b/Lib/test/test_multiprocessing.py @@ -1820,6 +1820,11 @@ class _TestZZZNumberOfObjects(BaseTestCase): # run after all the other tests for the manager. It tests that # there have been no "reference leaks" for the manager's shared # objects. Note the comment in _TestPool.test_terminate(). + + # If some other test using ManagerMixin.manager fails, then the + # raised exception may keep alive a frame which holds a reference + # to a managed object. This will cause test_number_of_objects to + # also fail. ALLOWED_TYPES = ('manager',) def test_number_of_objects(self): -- cgit v0.12 From 27f6a3b0bff9f100331707a4a461446ffc18baae Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 15 Jun 2012 22:15:23 +0200 Subject: Issue #15026: utf-16 encoding is now significantly faster (up to 10x). Patch by Serhiy Storchaka. --- Include/unicodeobject.h | 4 +-- Misc/NEWS | 3 ++ Objects/stringlib/codecs.h | 64 +++++++++++++++++++++++++++++++++++++ Objects/unicodeobject.c | 80 +++++++++++++++++++--------------------------- 4 files changed, 102 insertions(+), 49 deletions(-) diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index e40cc98..135e469 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -188,9 +188,9 @@ typedef unsigned char Py_UCS1; (((((Py_UCS4)(high) & 0x03FF) << 10) | \ ((Py_UCS4)(low) & 0x03FF)) + 0x10000) /* high surrogate = top 10 bits added to D800 */ -#define Py_UNICODE_HIGH_SURROGATE(ch) (0xD800 | (((ch) - 0x10000) >> 10)) +#define Py_UNICODE_HIGH_SURROGATE(ch) (0xD800 - (0x10000 >> 10) + ((ch) >> 10)) /* low surrogate = bottom 10 bits added to DC00 */ -#define Py_UNICODE_LOW_SURROGATE(ch) (0xDC00 | (((ch) - 0x10000) & 0x3FF)) +#define Py_UNICODE_LOW_SURROGATE(ch) (0xDC00 + ((ch) & 0x3FF)) /* Check if substring matches at given offset. The offset must be valid, and the substring must not be empty. */ diff --git a/Misc/NEWS b/Misc/NEWS index d701044..6acf02b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.3.0 Beta 1? Core and Builtins ----------------- +- Issue #15026: utf-16 encoding is now significantly faster (up to 10x). + Patch by Serhiy Storchaka. + - Issue #11022: open() and io.TextIOWrapper are now calling locale.getpreferredencoding(False) instead of locale.getpreferredencoding() in text mode if the encoding is not specified. Don't change temporary the diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index 07627d6..fb35493 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -562,4 +562,68 @@ IllegalSurrogate: #undef STRIPPED_MASK #undef SWAB #undef LONG_PTR_MASK + + +Py_LOCAL_INLINE(void) +STRINGLIB(utf16_encode)(unsigned short *out, + const STRINGLIB_CHAR *in, + Py_ssize_t len, + int native_ordering) +{ + const STRINGLIB_CHAR *end = in + len; +#if STRINGLIB_SIZEOF_CHAR == 1 +# define SWAB2(CH) ((CH) << 8) +#else +# define SWAB2(CH) (((CH) << 8) | ((CH) >> 8)) +#endif +#if STRINGLIB_MAX_CHAR < 0x10000 + if (native_ordering) { +# if STRINGLIB_SIZEOF_CHAR == 2 + Py_MEMCPY(out, in, 2 * len); +# else + _PyUnicode_CONVERT_BYTES(STRINGLIB_CHAR, unsigned short, in, end, out); +# endif + } else { + const STRINGLIB_CHAR *unrolled_end = in + (len & ~ (Py_ssize_t) 3); + while (in < unrolled_end) { + out[0] = SWAB2(in[0]); + out[1] = SWAB2(in[1]); + out[2] = SWAB2(in[2]); + out[3] = SWAB2(in[3]); + in += 4; out += 4; + } + while (in < end) { + *out++ = SWAB2(*in); + ++in; + } + } +#else + if (native_ordering) { + while (in < end) { + Py_UCS4 ch = *in++; + if (ch < 0x10000) + *out++ = ch; + else { + out[0] = Py_UNICODE_HIGH_SURROGATE(ch); + out[1] = Py_UNICODE_LOW_SURROGATE(ch); + out += 2; + } + } + } else { + while (in < end) { + Py_UCS4 ch = *in++; + if (ch < 0x10000) + *out++ = SWAB2((Py_UCS2)ch); + else { + Py_UCS2 ch1 = Py_UNICODE_HIGH_SURROGATE(ch); + Py_UCS2 ch2 = Py_UNICODE_LOW_SURROGATE(ch); + out[0] = SWAB2(ch1); + out[1] = SWAB2(ch2); + out += 2; + } + } + } +#endif +#undef SWAB2 +} #endif /* STRINGLIB_IS_UNICODE */ diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index a1efec0..c974ffe 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5359,27 +5359,19 @@ _PyUnicode_EncodeUTF16(PyObject *str, const char *errors, int byteorder) { - int kind; - void *data; + enum PyUnicode_Kind kind; + const void *data; Py_ssize_t len; PyObject *v; - unsigned char *p; - Py_ssize_t nsize, bytesize; - Py_ssize_t i, pairs; - /* Offsets from p for storing byte pairs in the right order. */ -#ifdef BYTEORDER_IS_LITTLE_ENDIAN - int ihi = 1, ilo = 0; + unsigned short *out; + Py_ssize_t bytesize; + Py_ssize_t pairs; +#ifdef WORDS_BIGENDIAN + int native_ordering = byteorder >= 0; #else - int ihi = 0, ilo = 1; + int native_ordering = byteorder <= 0; #endif -#define STORECHAR(CH) \ - do { \ - p[ihi] = ((CH) >> 8) & 0xff; \ - p[ilo] = (CH) & 0xff; \ - p += 2; \ - } while(0) - if (!PyUnicode_Check(str)) { PyErr_BadArgument(); return NULL; @@ -5391,53 +5383,47 @@ _PyUnicode_EncodeUTF16(PyObject *str, len = PyUnicode_GET_LENGTH(str); pairs = 0; - if (kind == PyUnicode_4BYTE_KIND) - for (i = 0; i < len; i++) - if (PyUnicode_READ(kind, data, i) >= 0x10000) + if (kind == PyUnicode_4BYTE_KIND) { + const Py_UCS4 *in = (const Py_UCS4 *)data; + const Py_UCS4 *end = in + len; + while (in < end) + if (*in++ >= 0x10000) pairs++; - /* 2 * (len + pairs + (byteorder == 0)) */ - if (len > PY_SSIZE_T_MAX - pairs - (byteorder == 0)) - return PyErr_NoMemory(); - nsize = len + pairs + (byteorder == 0); - bytesize = nsize * 2; - if (bytesize / 2 != nsize) + } + if (len > PY_SSIZE_T_MAX / 2 - pairs - (byteorder == 0)) return PyErr_NoMemory(); + bytesize = (len + pairs + (byteorder == 0)) * 2; v = PyBytes_FromStringAndSize(NULL, bytesize); if (v == NULL) return NULL; - p = (unsigned char *)PyBytes_AS_STRING(v); + /* output buffer is 2-bytes aligned */ + assert(((Py_uintptr_t)PyBytes_AS_STRING(v) & 1) == 0); + out = (unsigned short *)PyBytes_AS_STRING(v); if (byteorder == 0) - STORECHAR(0xFEFF); + *out++ = 0xFEFF; if (len == 0) goto done; - if (byteorder == -1) { - /* force LE */ - ihi = 1; - ilo = 0; + switch (kind) { + case PyUnicode_1BYTE_KIND: { + ucs1lib_utf16_encode(out, (const Py_UCS1 *)data, len, native_ordering); + break; } - else if (byteorder == 1) { - /* force BE */ - ihi = 0; - ilo = 1; + case PyUnicode_2BYTE_KIND: { + ucs2lib_utf16_encode(out, (const Py_UCS2 *)data, len, native_ordering); + break; } - - for (i = 0; i < len; i++) { - Py_UCS4 ch = PyUnicode_READ(kind, data, i); - Py_UCS4 ch2 = 0; - if (ch >= 0x10000) { - ch2 = Py_UNICODE_LOW_SURROGATE(ch); - ch = Py_UNICODE_HIGH_SURROGATE(ch); - } - STORECHAR(ch); - if (ch2) - STORECHAR(ch2); + case PyUnicode_4BYTE_KIND: { + ucs4lib_utf16_encode(out, (const Py_UCS4 *)data, len, native_ordering); + break; + } + default: + assert(0); } done: return v; -#undef STORECHAR } PyObject * -- cgit v0.12 From c90929624b53561a37c730a7801bd14b1d2873ca Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 15 Jun 2012 22:22:18 +0200 Subject: Mention the UTF-16 encoding speedup in the whatsnew (issue #15026). --- Doc/whatsnew/3.3.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index f52d5ae..974cc71 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -1484,9 +1484,11 @@ Major performance enhancements have been added: * repeating a single ASCII letter and getting a substring of a ASCII strings is 4 times faster -* UTF-8 and UTF-16 decoding is now 2x to 4x faster. +* UTF-8 and UTF-16 decoding is now 2x to 4x faster. UTF-16 encoding is now + up to 10x faster. - (contributed by Serhiy Storchaka, :issue:`14624` and :issue:`14738`.) + (contributed by Serhiy Storchaka, :issue:`14624`, :issue:`14738` and + :issue:`15026`.) Build and C API Changes -- cgit v0.12 From 9125775aa6f98541c5357e0153bd5fd067353469 Mon Sep 17 00:00:00 2001 From: Richard Oudkerk Date: Fri, 15 Jun 2012 21:53:34 +0100 Subject: Fix _TestListener.ALLOWED_TYPES and add sanity check --- Lib/test/test_multiprocessing.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py index d79110a..2704827 100644 --- a/Lib/test/test_multiprocessing.py +++ b/Lib/test/test_multiprocessing.py @@ -2268,7 +2268,7 @@ class _TestConnection(BaseTestCase): class _TestListener(BaseTestCase): - ALLOWED_TYPES = ('processes') + ALLOWED_TYPES = ('processes',) def test_multiple_bind(self): for family in self.connection.families: @@ -2850,10 +2850,12 @@ def create_test_cases(Mixin, type): result = {} glob = globals() Type = type.capitalize() + ALL_TYPES = {'processes', 'threads', 'manager'} for name in list(glob.keys()): if name.startswith('_Test'): base = glob[name] + assert set(base.ALLOWED_TYPES) <= ALL_TYPES, set(base.ALLOWED_TYPES) if type in base.ALLOWED_TYPES: newname = 'With' + Type + name[1:] class Temp(base, unittest.TestCase, Mixin): -- cgit v0.12 From 016ef551a793f72f582d707ce5bb55bf4940cf27 Mon Sep 17 00:00:00 2001 From: Alexander Belopolsky Date: Fri, 15 Jun 2012 18:15:25 -0400 Subject: Removed redundant code --- Lib/datetime.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Lib/datetime.py b/Lib/datetime.py index 5d8d9b3..21aab35 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1670,10 +1670,8 @@ class datetime(date): if mytz is ottz: base_compare = True else: - if mytz is not None: - myoff = self.utcoffset() - if ottz is not None: - otoff = other.utcoffset() + myoff = self.utcoffset() + otoff = other.utcoffset() base_compare = myoff == otoff if base_compare: -- cgit v0.12