diff options
author | Brett Cannon <brett@python.org> | 2012-06-15 23:04:29 (GMT) |
---|---|---|
committer | Brett Cannon <brett@python.org> | 2012-06-15 23:04:29 (GMT) |
commit | 24aa693c7ef8f217fbd238eb7af7d828e13a07eb (patch) | |
tree | 7d01ab630c2e8eef1e168b1aa5d84131b60cfd50 | |
parent | 99d776fdf4aa5a66266ebcec2263fab501f03088 (diff) | |
parent | 016ef551a793f72f582d707ce5bb55bf4940cf27 (diff) | |
download | cpython-24aa693c7ef8f217fbd238eb7af7d828e13a07eb.zip cpython-24aa693c7ef8f217fbd238eb7af7d828e13a07eb.tar.gz cpython-24aa693c7ef8f217fbd238eb7af7d828e13a07eb.tar.bz2 |
Merge
38 files changed, 1358 insertions, 609 deletions
@@ -55,6 +55,8 @@ PC/python_nt*.h PC/pythonnt_rc*.h PC/*.obj PC/*.exe +PC/*/*.exe +PC/*/*.pdb PC/*/*.user PC/*/*.ncb PC/*/*.suo diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 76c55fc..b90d35c 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -1437,9 +1437,8 @@ The type constructor is responsible for initializing the weak reference list to } The only further addition is that the destructor needs to call the weak -reference manager to clear any weak references. This should be done before any -other parts of the destruction have occurred, but is only required if the weak -reference list is non-*NULL*:: +reference manager to clear any weak references. This is only required if the +weak reference list is non-*NULL*:: static void instance_dealloc(PyInstanceObject *inst) diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index 8274bb1..e6ce99b 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -42,8 +42,8 @@ An HMAC object has the following methods: When comparing the output of :meth:`digest` to an externally-supplied digest during a verification routine, it is recommended to use the - :func:`hmac.secure_compare` function instead of the ``==`` operator - to avoid potential timing attacks. + :func:`compare_digest` function instead of the ``==`` operator + to reduce the vulnerability to timing attacks. .. method:: HMAC.hexdigest() @@ -54,10 +54,11 @@ An HMAC object has the following methods: .. warning:: - When comparing the output of :meth:`hexdigest` to an externally-supplied - digest during a verification routine, it is recommended to use the - :func:`hmac.secure_compare` function instead of the ``==`` operator - to avoid potential timing attacks. + The output of :meth:`hexdigest` should not be compared directly to an + externally-supplied digest during a verification routine. Instead, the + externally supplied digest should be converted to a :class:`bytes` + value and compared to the output of :meth:`digest` with + :func:`compare_digest`. .. method:: HMAC.copy() @@ -68,20 +69,28 @@ An HMAC object has the following methods: This module also provides the following helper function: -.. function:: secure_compare(a, b) +.. function:: compare_digest(a, b) + + Returns the equivalent of ``a == b``, but avoids content based + short circuiting behaviour to reduce the vulnerability to timing + analysis. The inputs must be :class:`bytes` instances. - Returns the equivalent of ``a == b``, but using a time-independent - comparison method. Comparing the full lengths of the inputs *a* and *b*, - instead of short-circuiting the comparison upon the first unequal byte, - prevents leaking information about the inputs being compared and mitigates - potential timing attacks. The inputs must be either :class:`str` or - :class:`bytes` instances. + Using a short circuiting comparison (that is, one that terminates as soon + as it finds any difference between the values) to check digests for + correctness can be problematic, as it introduces a potential + vulnerability when an attacker can control both the message to be checked + *and* the purported signature value. By keeping the plaintext consistent + and supplying different signature values, an attacker may be able to use + timing variations to search the signature space for the expected value in + O(n) time rather than the desired O(2**n). .. note:: - While the :func:`hmac.secure_compare` function prevents leaking the - contents of the inputs via a timing attack, it does leak the length - of the inputs. However, this generally is not a security risk. + While this function reduces the likelihood of leaking the contents of + the expected digest via a timing attack, it still uses short circuiting + behaviour based on the *length* of the inputs. It is assumed that the + expected length of the digest is not a secret, as it is typically + published as part of a file format, network protocol or API definition. .. versionadded:: 3.3 diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 43ae086..4171977 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -226,11 +226,11 @@ However, if you really do need to use some shared data then holds Python objects and allows other processes to manipulate them using proxies. - A manager returned by :func:`Manager` will support types :class:`list`, - :class:`dict`, :class:`Namespace`, :class:`Lock`, :class:`RLock`, - :class:`Semaphore`, :class:`BoundedSemaphore`, :class:`Condition`, - :class:`Event`, :class:`Queue`, :class:`Value` and :class:`Array`. For - example, :: + A manager returned by :func:`Manager` will support types + :class:`list`, :class:`dict`, :class:`Namespace`, :class:`Lock`, + :class:`RLock`, :class:`Semaphore`, :class:`BoundedSemaphore`, + :class:`Condition`, :class:`Event`, :class:`Barrier`, + :class:`Queue`, :class:`Value` and :class:`Array`. For example, :: from multiprocessing import Process, Manager @@ -885,6 +885,12 @@ program as they are in a multithreaded program. See the documentation for Note that one can also create synchronization primitives by using a manager object -- see :ref:`multiprocessing-managers`. +.. class:: Barrier(parties[, action[, timeout]]) + + A barrier object: a clone of :class:`threading.Barrier`. + + .. versionadded:: 3.3 + .. class:: BoundedSemaphore([value]) A bounded semaphore object: a clone of :class:`threading.BoundedSemaphore`. @@ -1236,9 +1242,10 @@ their parent process exits. The manager classes are defined in the type of shared object. This must be a string. *callable* is a callable used for creating objects for this type - identifier. If a manager instance will be created using the - :meth:`from_address` classmethod or if the *create_method* argument is - ``False`` then this can be left as ``None``. + identifier. If a manager instance will be connected to the + server using the :meth:`connect` method, or if the + *create_method* argument is ``False`` then this can be left as + ``None``. *proxytype* is a subclass of :class:`BaseProxy` which is used to create proxies for shared objects with this *typeid*. If ``None`` then a proxy @@ -1279,6 +1286,13 @@ their parent process exits. The manager classes are defined in the It also supports creation of shared lists and dictionaries. + .. method:: Barrier(parties[, action[, timeout]]) + + Create a shared :class:`threading.Barrier` object and return a + proxy for it. + + .. versionadded:: 3.3 + .. method:: BoundedSemaphore([value]) Create a shared :class:`threading.BoundedSemaphore` object and return a diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 0315507..02af9ca 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -61,7 +61,7 @@ created. Socket addresses are represented as follows: - A pair ``(host, port)`` is used for the :const:`AF_INET` address family, where *host* is a string representing either a hostname in Internet domain notation like ``'daring.cwi.nl'`` or an IPv4 address like ``'100.50.200.5'``, - and *port* is an integral port number. + and *port* is an integer. - For :const:`AF_INET6` address family, a four-tuple ``(host, port, flowinfo, scopeid)`` is used, where *flowinfo* and *scopeid* represent the ``sin6_flowinfo`` diff --git a/Doc/library/time.rst b/Doc/library/time.rst index a8c70b3..3faabf7 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -77,6 +77,12 @@ An explanation of some terminology and conventions is in order. See :class:`struct_time` for a description of these objects. + .. versionchanged:: 3.3 + + The :class:`struct_time` type was extended to provide the + :attr:`tm_gmtoff` and :attr:`tm_zone` attributes when platform + supports corresponding ``struct tm`` members. + * Use the following functions to convert between time representations: +-------------------------+-------------------------+-------------------------+ @@ -160,30 +166,6 @@ The module defines the following functions and data items: .. versionadded:: 3.3 -.. class:: clock_info - - Clock information object returned by :func:`get_clock_info`. - - .. attribute:: implementation - - The name of the underlying C function used to get the clock value. - - .. attribute:: monotonic - - ``True`` if the clock cannot go backward, ``False`` otherwise. - - .. attribute:: adjusted - - ``True`` if the clock can be adjusted (e.g. by a NTP daemon), ``False`` - otherwise. - - .. attribute:: resolution - - The resolution of the clock in seconds (:class:`float`). - - .. versionadded:: 3.3 - - .. function:: clock_settime(clk_id, time) Set the time of the specified clock *clk_id*. @@ -267,7 +249,7 @@ The module defines the following functions and data items: .. function:: get_clock_info(name) - Get information on the specified clock as a :class:`clock_info` object. + Get information on the specified clock as a namespace object. Supported clock names and the corresponding functions to read their value are: @@ -277,6 +259,16 @@ The module defines the following functions and data items: * ``'process_time'``: :func:`time.process_time` * ``'time'``: :func:`time.time` + The result has the following attributes: + + - *adjustable*: ``True`` if the clock can be changed automatically (e.g. by + a NTP daemon) or manually by the system administrator, ``False`` otherwise + - *implementation*: The name of the underlying C function used to get + the clock value + - *monotonic*: ``True`` if the clock cannot go backward, + ``False`` otherwise + - *resolution*: The resolution of the clock in seconds (:class:`float`) + .. versionadded:: 3.3 @@ -350,7 +342,6 @@ The module defines the following functions and data items: .. versionadded:: 3.3 - .. function:: sleep(secs) Suspend execution for the given number of seconds. The argument may be a @@ -447,6 +438,12 @@ The module defines the following functions and data items: | ``%Y`` | Year with century as a decimal number. | | | | | | +-----------+------------------------------------------------+-------+ + | ``%z`` | Time zone offset indicating a positive or | | + | | negative time difference from UTC/GMT of the | | + | | form +HHMM or -HHMM, where H represents decimal| | + | | hour digits and M represents decimal minute | | + | | digits [-23:59, +23:59]. | | + +-----------+------------------------------------------------+-------+ | ``%Z`` | Time zone name (no characters if no time zone | | | | exists). | | +-----------+------------------------------------------------+-------+ @@ -546,6 +543,10 @@ The module defines the following functions and data items: +-------+-------------------+---------------------------------+ | 8 | :attr:`tm_isdst` | 0, 1 or -1; see below | +-------+-------------------+---------------------------------+ + | N/A | :attr:`tm_zone` | abbreviation of timezone name | + +-------+-------------------+---------------------------------+ + | N/A | :attr:`tm_gmtoff` | offset from UTC in seconds | + +-------+-------------------+---------------------------------+ Note that unlike the C structure, the month value is a range of [1, 12], not [0, 11]. A ``-1`` argument as the daylight @@ -556,6 +557,11 @@ The module defines the following functions and data items: :class:`struct_time`, or having elements of the wrong type, a :exc:`TypeError` is raised. + .. versionchanged:: 3.3 + + :attr:`tm_gmtoff` and :attr:`tm_zone` attributes are avaliable on + platforms with C library supporting the corresponding fields in + ``struct tm``. .. function:: time() @@ -566,7 +572,6 @@ The module defines the following functions and data items: lower value than a previous call if the system clock has been set back between the two calls. - .. data:: timezone The offset of the local (non-DST) timezone, in seconds west of UTC (negative in diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index f52d5ae..974cc71 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -1484,9 +1484,11 @@ Major performance enhancements have been added: * repeating a single ASCII letter and getting a substring of a ASCII strings is 4 times faster -* UTF-8 and UTF-16 decoding is now 2x to 4x faster. +* UTF-8 and UTF-16 decoding is now 2x to 4x faster. UTF-16 encoding is now + up to 10x faster. - (contributed by Serhiy Storchaka, :issue:`14624` and :issue:`14738`.) + (contributed by Serhiy Storchaka, :issue:`14624`, :issue:`14738` and + :issue:`15026`.) Build and C API Changes diff --git a/Include/pytime.h b/Include/pytime.h index dd4cc69..52902f5 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -26,7 +26,7 @@ typedef struct { typedef struct { const char *implementation; int monotonic; - int adjusted; + int adjustable; double resolution; } _Py_clock_info_t; diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index e40cc98..135e469 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -188,9 +188,9 @@ typedef unsigned char Py_UCS1; (((((Py_UCS4)(high) & 0x03FF) << 10) | \ ((Py_UCS4)(low) & 0x03FF)) + 0x10000) /* high surrogate = top 10 bits added to D800 */ -#define Py_UNICODE_HIGH_SURROGATE(ch) (0xD800 | (((ch) - 0x10000) >> 10)) +#define Py_UNICODE_HIGH_SURROGATE(ch) (0xD800 - (0x10000 >> 10) + ((ch) >> 10)) /* low surrogate = bottom 10 bits added to DC00 */ -#define Py_UNICODE_LOW_SURROGATE(ch) (0xDC00 | (((ch) - 0x10000) & 0x3FF)) +#define Py_UNICODE_LOW_SURROGATE(ch) (0xDC00 + ((ch) & 0x3FF)) /* Check if substring matches at given offset. The offset must be valid, and the substring must not be empty. */ diff --git a/Lib/_strptime.py b/Lib/_strptime.py index fa06376..b0cd3d6 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -486,19 +486,19 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): return (year, month, day, hour, minute, second, - weekday, julian, tz, gmtoff, tzname), fraction + weekday, julian, tz, tzname, gmtoff), fraction def _strptime_time(data_string, format="%a %b %d %H:%M:%S %Y"): """Return a time struct based on the input string and the format string.""" tt = _strptime(data_string, format)[0] - return time.struct_time(tt[:9]) + return time.struct_time(tt[:time._STRUCT_TM_ITEMS]) def _strptime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"): """Return a class cls instance based on the input string and the format string.""" tt, fraction = _strptime(data_string, format) - gmtoff, tzname = tt[-2:] + tzname, gmtoff = tt[-2:] args = tt[:6] + (fraction,) if gmtoff is not None: tzdelta = datetime_timedelta(seconds=gmtoff) diff --git a/Lib/datetime.py b/Lib/datetime.py index 5d8d9b3..21aab35 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1670,10 +1670,8 @@ class datetime(date): if mytz is ottz: base_compare = True else: - if mytz is not None: - myoff = self.utcoffset() - if ottz is not None: - otoff = other.utcoffset() + myoff = self.utcoffset() + otoff = other.utcoffset() base_compare = myoff == otoff if base_compare: diff --git a/Lib/hmac.py b/Lib/hmac.py index 13ffdbe..e47965b 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -13,24 +13,24 @@ trans_36 = bytes((x ^ 0x36) for x in range(256)) digest_size = None -def secure_compare(a, b): - """Returns the equivalent of 'a == b', but using a time-independent - comparison method to prevent timing attacks.""" - if not ((isinstance(a, str) and isinstance(b, str)) or - (isinstance(a, bytes) and isinstance(b, bytes))): - raise TypeError("inputs must be strings or bytes") - +def compare_digest(a, b): + """Returns the equivalent of 'a == b', but avoids content based short + circuiting to reduce the vulnerability to timing attacks.""" + # Consistent timing matters more here than data type flexibility + if not (isinstance(a, bytes) and isinstance(b, bytes)): + raise TypeError("inputs must be bytes instances") + + # We assume the length of the expected digest is public knowledge, + # thus this early return isn't leaking anything an attacker wouldn't + # already know if len(a) != len(b): return False + # We assume that integers in the bytes range are all cached, + # thus timing shouldn't vary much due to integer object creation result = 0 - if isinstance(a, bytes): - for x, y in zip(a, b): - result |= x ^ y - else: - for x, y in zip(a, b): - result |= ord(x) ^ ord(y) - + for x, y in zip(a, b): + result |= x ^ y return result == 0 diff --git a/Lib/idlelib/AutoComplete.py b/Lib/idlelib/AutoComplete.py index b38b108..929d358 100644 --- a/Lib/idlelib/AutoComplete.py +++ b/Lib/idlelib/AutoComplete.py @@ -140,7 +140,7 @@ class AutoComplete: elif hp.is_in_code() and (not mode or mode==COMPLETE_ATTRIBUTES): self._remove_autocomplete_window() mode = COMPLETE_ATTRIBUTES - while i and curline[i-1] in ID_CHARS or ord(curline[i-1]) > 127: + while i and (curline[i-1] in ID_CHARS or ord(curline[i-1]) > 127): i -= 1 comp_start = curline[i:j] if i and curline[i-1] == '.': diff --git a/Lib/mailbox.py b/Lib/mailbox.py index 7a29555..d86ad94 100644 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -675,6 +675,7 @@ class _singlefileMailbox(Mailbox): new_file.write(buffer) new_toc[key] = (new_start, new_file.tell()) self._post_message_hook(new_file) + self._file_length = new_file.tell() except: new_file.close() os.remove(new_file.name) diff --git a/Lib/multiprocessing/__init__.py b/Lib/multiprocessing/__init__.py index 02460f0..1f3e67c 100644 --- a/Lib/multiprocessing/__init__.py +++ b/Lib/multiprocessing/__init__.py @@ -23,8 +23,8 @@ __all__ = [ 'Manager', 'Pipe', 'cpu_count', 'log_to_stderr', 'get_logger', 'allow_connection_pickling', 'BufferTooShort', 'TimeoutError', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', - 'Event', 'Queue', 'SimpleQueue', 'JoinableQueue', 'Pool', 'Value', 'Array', - 'RawValue', 'RawArray', 'SUBDEBUG', 'SUBWARNING', + 'Event', 'Barrier', 'Queue', 'SimpleQueue', 'JoinableQueue', 'Pool', + 'Value', 'Array', 'RawValue', 'RawArray', 'SUBDEBUG', 'SUBWARNING', ] __author__ = 'R. Oudkerk (r.m.oudkerk@gmail.com)' @@ -186,6 +186,13 @@ def Event(): from multiprocessing.synchronize import Event return Event() +def Barrier(parties, action=None, timeout=None): + ''' + Returns a barrier object + ''' + from multiprocessing.synchronize import Barrier + return Barrier(parties, action, timeout) + def Queue(maxsize=0): ''' Returns a queue object diff --git a/Lib/multiprocessing/dummy/__init__.py b/Lib/multiprocessing/dummy/__init__.py index 9bf8f6b..e31fc61 100644 --- a/Lib/multiprocessing/dummy/__init__.py +++ b/Lib/multiprocessing/dummy/__init__.py @@ -35,7 +35,7 @@ __all__ = [ 'Process', 'current_process', 'active_children', 'freeze_support', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', - 'Event', 'Queue', 'Manager', 'Pipe', 'Pool', 'JoinableQueue' + 'Event', 'Barrier', 'Queue', 'Manager', 'Pipe', 'Pool', 'JoinableQueue' ] # @@ -49,7 +49,7 @@ import array from multiprocessing.dummy.connection import Pipe from threading import Lock, RLock, Semaphore, BoundedSemaphore -from threading import Event, Condition +from threading import Event, Condition, Barrier from queue import Queue # diff --git a/Lib/multiprocessing/forking.py b/Lib/multiprocessing/forking.py index 3a474cd..4baf548 100644 --- a/Lib/multiprocessing/forking.py +++ b/Lib/multiprocessing/forking.py @@ -13,7 +13,7 @@ import signal from multiprocessing import util, process -__all__ = ['Popen', 'assert_spawning', 'exit', 'duplicate', 'close', 'ForkingPickler'] +__all__ = ['Popen', 'assert_spawning', 'duplicate', 'close', 'ForkingPickler'] # # Check that the current thread is spawning a child process @@ -75,7 +75,6 @@ else: # if sys.platform != 'win32': - exit = os._exit duplicate = os.dup close = os.close @@ -168,7 +167,6 @@ else: WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False)) WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") - exit = _winapi.ExitProcess close = _winapi.CloseHandle # @@ -349,7 +347,7 @@ else: from_parent.close() exitcode = self._bootstrap() - exit(exitcode) + sys.exit(exitcode) def get_preparation_data(name): diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 6a7dccb..f6611af 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -22,7 +22,7 @@ import queue from traceback import format_exc from multiprocessing import Process, current_process, active_children, Pool, util, connection from multiprocessing.process import AuthenticationString -from multiprocessing.forking import exit, Popen, ForkingPickler +from multiprocessing.forking import Popen, ForkingPickler from time import time as _time # @@ -140,28 +140,38 @@ class Server(object): self.id_to_obj = {'0': (None, ())} self.id_to_refcount = {} self.mutex = threading.RLock() - self.stop = 0 def serve_forever(self): ''' Run the server forever ''' + self.stop_event = threading.Event() current_process()._manager_server = self try: + accepter = threading.Thread(target=self.accepter) + accepter.daemon = True + accepter.start() try: - while 1: - try: - c = self.listener.accept() - except (OSError, IOError): - continue - t = threading.Thread(target=self.handle_request, args=(c,)) - t.daemon = True - t.start() + while not self.stop_event.is_set(): + self.stop_event.wait(1) except (KeyboardInterrupt, SystemExit): pass finally: - self.stop = 999 - self.listener.close() + if sys.stdout != sys.__stdout__: + util.debug('resetting stdout, stderr') + sys.stdout = sys.__stdout__ + sys.stderr = sys.__stderr__ + sys.exit(0) + + def accepter(self): + while True: + try: + c = self.listener.accept() + except (OSError, IOError): + continue + t = threading.Thread(target=self.handle_request, args=(c,)) + t.daemon = True + t.start() def handle_request(self, c): ''' @@ -208,7 +218,7 @@ class Server(object): send = conn.send id_to_obj = self.id_to_obj - while not self.stop: + while not self.stop_event.is_set(): try: methodname = obj = None @@ -318,32 +328,13 @@ class Server(object): Shutdown this process ''' try: - try: - util.debug('manager received shutdown message') - c.send(('#RETURN', None)) - - if sys.stdout != sys.__stdout__: - util.debug('resetting stdout, stderr') - sys.stdout = sys.__stdout__ - sys.stderr = sys.__stderr__ - - util._run_finalizers(0) - - for p in active_children(): - util.debug('terminating a child process of manager') - p.terminate() - - for p in active_children(): - util.debug('terminating a child process of manager') - p.join() - - util._run_finalizers() - util.info('manager exiting with exitcode 0') - except: - import traceback - traceback.print_exc() + util.debug('manager received shutdown message') + c.send(('#RETURN', None)) + except: + import traceback + traceback.print_exc() finally: - exit(0) + self.stop_event.set() def create(self, c, typeid, *args, **kwds): ''' @@ -455,10 +446,6 @@ class BaseManager(object): self._serializer = serializer self._Listener, self._Client = listener_client[serializer] - def __reduce__(self): - return type(self).from_address, \ - (self._address, self._authkey, self._serializer) - def get_server(self): ''' Return server object with serve_forever() method and address attribute @@ -595,7 +582,7 @@ class BaseManager(object): except Exception: pass - process.join(timeout=0.2) + process.join(timeout=1.0) if process.is_alive(): util.info('manager still alive') if hasattr(process, 'terminate'): @@ -1006,6 +993,26 @@ class EventProxy(BaseProxy): def wait(self, timeout=None): return self._callmethod('wait', (timeout,)) + +class BarrierProxy(BaseProxy): + _exposed_ = ('__getattribute__', 'wait', 'abort', 'reset') + def wait(self, timeout=None): + return self._callmethod('wait', (timeout,)) + def abort(self): + return self._callmethod('abort') + def reset(self): + return self._callmethod('reset') + @property + def parties(self): + return self._callmethod('__getattribute__', ('parties',)) + @property + def n_waiting(self): + return self._callmethod('__getattribute__', ('n_waiting',)) + @property + def broken(self): + return self._callmethod('__getattribute__', ('broken',)) + + class NamespaceProxy(BaseProxy): _exposed_ = ('__getattribute__', '__setattr__', '__delattr__') def __getattr__(self, key): @@ -1097,6 +1104,7 @@ SyncManager.register('Semaphore', threading.Semaphore, AcquirerProxy) SyncManager.register('BoundedSemaphore', threading.BoundedSemaphore, AcquirerProxy) SyncManager.register('Condition', threading.Condition, ConditionProxy) +SyncManager.register('Barrier', threading.Barrier, BarrierProxy) SyncManager.register('Pool', Pool, PoolProxy) SyncManager.register('list', list, ListProxy) SyncManager.register('dict', dict, DictProxy) diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index 4502a97..22eabe5 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -333,3 +333,43 @@ class Event(object): return False finally: self._cond.release() + +# +# Barrier +# + +class Barrier(threading.Barrier): + + def __init__(self, parties, action=None, timeout=None): + import struct + from multiprocessing.heap import BufferWrapper + wrapper = BufferWrapper(struct.calcsize('i') * 2) + cond = Condition() + self.__setstate__((parties, action, timeout, cond, wrapper)) + self._state = 0 + self._count = 0 + + def __setstate__(self, state): + (self._parties, self._action, self._timeout, + self._cond, self._wrapper) = state + self._array = self._wrapper.create_memoryview().cast('i') + + def __getstate__(self): + return (self._parties, self._action, self._timeout, + self._cond, self._wrapper) + + @property + def _state(self): + return self._array[0] + + @_state.setter + def _state(self, value): + self._array[0] = value + + @property + def _count(self): + return self._array[1] + + @_count.setter + def _count(self, value): + self._array[1] = value diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 48abe38..8a6aede 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -269,21 +269,24 @@ _exiting = False def _exit_function(): global _exiting - info('process shutting down') - debug('running all "atexit" finalizers with priority >= 0') - _run_finalizers(0) + if not _exiting: + _exiting = True - for p in active_children(): - if p._daemonic: - info('calling terminate() for daemon %s', p.name) - p._popen.terminate() + info('process shutting down') + debug('running all "atexit" finalizers with priority >= 0') + _run_finalizers(0) - for p in active_children(): - info('calling join() for process %s', p.name) - p.join() + for p in active_children(): + if p._daemonic: + info('calling terminate() for daemon %s', p.name) + p._popen.terminate() - debug('running the remaining "atexit" finalizers') - _run_finalizers() + for p in active_children(): + info('calling join() for process %s', p.name) + p.join() + + debug('running the remaining "atexit" finalizers') + _run_finalizers() atexit.register(_exit_function) diff --git a/Lib/test/support.py b/Lib/test/support.py index 6749a511..3ff1df5 100644 --- a/Lib/test/support.py +++ b/Lib/test/support.py @@ -1593,7 +1593,7 @@ def strip_python_stderr(stderr): This will typically be run on the result of the communicate() method of a subprocess.Popen object. """ - stderr = re.sub(br"\[\d+ refs\]\r?\n?$", b"", stderr).strip() + stderr = re.sub(br"\[\d+ refs\]\r?\n?", b"", stderr).strip() return stderr def args_from_interpreter_flags(): diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index 042bc5d..4e5961d 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -302,40 +302,42 @@ class CopyTestCase(unittest.TestCase): self.assertEqual(h1.hexdigest(), h2.hexdigest(), "Hexdigest of copy doesn't match original hexdigest.") -class SecureCompareTestCase(unittest.TestCase): +class CompareDigestTestCase(unittest.TestCase): def test_compare(self): # Testing input type exception handling a, b = 100, 200 - self.assertRaises(TypeError, hmac.secure_compare, a, b) - a, b = 100, "foobar" - self.assertRaises(TypeError, hmac.secure_compare, a, b) + self.assertRaises(TypeError, hmac.compare_digest, a, b) + a, b = 100, b"foobar" + self.assertRaises(TypeError, hmac.compare_digest, a, b) + a, b = b"foobar", 200 + self.assertRaises(TypeError, hmac.compare_digest, a, b) a, b = "foobar", b"foobar" - self.assertRaises(TypeError, hmac.secure_compare, a, b) + self.assertRaises(TypeError, hmac.compare_digest, a, b) + a, b = b"foobar", "foobar" + self.assertRaises(TypeError, hmac.compare_digest, a, b) + a, b = "foobar", "foobar" + self.assertRaises(TypeError, hmac.compare_digest, a, b) + a, b = bytearray(b"foobar"), bytearray(b"foobar") + self.assertRaises(TypeError, hmac.compare_digest, a, b) - # Testing str/bytes of different lengths - a, b = "foobar", "foo" - self.assertFalse(hmac.secure_compare(a, b)) + # Testing bytes of different lengths a, b = b"foobar", b"foo" - self.assertFalse(hmac.secure_compare(a, b)) + self.assertFalse(hmac.compare_digest(a, b)) a, b = b"\xde\xad\xbe\xef", b"\xde\xad" - self.assertFalse(hmac.secure_compare(a, b)) + self.assertFalse(hmac.compare_digest(a, b)) - # Testing str/bytes of same lengths, different values - a, b = "foobar", "foobaz" - self.assertFalse(hmac.secure_compare(a, b)) + # Testing bytes of same lengths, different values a, b = b"foobar", b"foobaz" - self.assertFalse(hmac.secure_compare(a, b)) + self.assertFalse(hmac.compare_digest(a, b)) a, b = b"\xde\xad\xbe\xef", b"\xab\xad\x1d\xea" - self.assertFalse(hmac.secure_compare(a, b)) + self.assertFalse(hmac.compare_digest(a, b)) - # Testing str/bytes of same lengths, same values - a, b = "foobar", "foobar" - self.assertTrue(hmac.secure_compare(a, b)) + # Testing bytes of same lengths, same values a, b = b"foobar", b"foobar" - self.assertTrue(hmac.secure_compare(a, b)) + self.assertTrue(hmac.compare_digest(a, b)) a, b = b"\xde\xad\xbe\xef", b"\xde\xad\xbe\xef" - self.assertTrue(hmac.secure_compare(a, b)) + self.assertTrue(hmac.compare_digest(a, b)) def test_main(): support.run_unittest( @@ -343,7 +345,7 @@ def test_main(): ConstructorTestCase, SanityTestCase, CopyTestCase, - SecureCompareTestCase + CompareDigestTestCase ) if __name__ == "__main__": diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 0fe4bd9..cb54505 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -504,6 +504,17 @@ class TestMailbox(TestBase): # Write changes to disk self._test_flush_or_close(self._box.flush, True) + def test_popitem_and_flush_twice(self): + # See #15036. + self._box.add(self._template % 0) + self._box.add(self._template % 1) + self._box.flush() + + self._box.popitem() + self._box.flush() + self._box.popitem() + self._box.flush() + def test_lock_unlock(self): # Lock and unlock the mailbox self.assertFalse(os.path.exists(self._get_lock_path())) diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py index 65e7b0b..2704827 100644 --- a/Lib/test/test_multiprocessing.py +++ b/Lib/test/test_multiprocessing.py @@ -18,6 +18,7 @@ import array import socket import random import logging +import struct import test.support @@ -1057,6 +1058,340 @@ class _TestEvent(BaseTestCase): self.assertEqual(wait(), True) # +# Tests for Barrier - adapted from tests in test/lock_tests.py +# + +# Many of the tests for threading.Barrier use a list as an atomic +# counter: a value is appended to increment the counter, and the +# length of the list gives the value. We use the class DummyList +# for the same purpose. + +class _DummyList(object): + + def __init__(self): + wrapper = multiprocessing.heap.BufferWrapper(struct.calcsize('i')) + lock = multiprocessing.Lock() + self.__setstate__((wrapper, lock)) + self._lengthbuf[0] = 0 + + def __setstate__(self, state): + (self._wrapper, self._lock) = state + self._lengthbuf = self._wrapper.create_memoryview().cast('i') + + def __getstate__(self): + return (self._wrapper, self._lock) + + def append(self, _): + with self._lock: + self._lengthbuf[0] += 1 + + def __len__(self): + with self._lock: + return self._lengthbuf[0] + +def _wait(): + # A crude wait/yield function not relying on synchronization primitives. + time.sleep(0.01) + + +class Bunch(object): + """ + A bunch of threads. + """ + def __init__(self, namespace, f, args, n, wait_before_exit=False): + """ + Construct a bunch of `n` threads running the same function `f`. + If `wait_before_exit` is True, the threads won't terminate until + do_finish() is called. + """ + self.f = f + self.args = args + self.n = n + self.started = namespace.DummyList() + self.finished = namespace.DummyList() + self._can_exit = namespace.Event() + if not wait_before_exit: + self._can_exit.set() + for i in range(n): + p = namespace.Process(target=self.task) + p.daemon = True + p.start() + + def task(self): + pid = os.getpid() + self.started.append(pid) + try: + self.f(*self.args) + finally: + self.finished.append(pid) + self._can_exit.wait(30) + assert self._can_exit.is_set() + + def wait_for_started(self): + while len(self.started) < self.n: + _wait() + + def wait_for_finished(self): + while len(self.finished) < self.n: + _wait() + + def do_finish(self): + self._can_exit.set() + + +class AppendTrue(object): + def __init__(self, obj): + self.obj = obj + def __call__(self): + self.obj.append(True) + + +class _TestBarrier(BaseTestCase): + """ + Tests for Barrier objects. + """ + N = 5 + defaultTimeout = 10.0 # XXX Slow Windows buildbots need generous timeout + + def setUp(self): + self.barrier = self.Barrier(self.N, timeout=self.defaultTimeout) + + def tearDown(self): + self.barrier.abort() + self.barrier = None + + def DummyList(self): + if self.TYPE == 'threads': + return [] + elif self.TYPE == 'manager': + return self.manager.list() + else: + return _DummyList() + + def run_threads(self, f, args): + b = Bunch(self, f, args, self.N-1) + f(*args) + b.wait_for_finished() + + @classmethod + def multipass(cls, barrier, results, n): + m = barrier.parties + assert m == cls.N + for i in range(n): + results[0].append(True) + assert len(results[1]) == i * m + barrier.wait() + results[1].append(True) + assert len(results[0]) == (i + 1) * m + barrier.wait() + try: + assert barrier.n_waiting == 0 + except NotImplementedError: + pass + assert not barrier.broken + + def test_barrier(self, passes=1): + """ + Test that a barrier is passed in lockstep + """ + results = [self.DummyList(), self.DummyList()] + self.run_threads(self.multipass, (self.barrier, results, passes)) + + def test_barrier_10(self): + """ + Test that a barrier works for 10 consecutive runs + """ + return self.test_barrier(10) + + @classmethod + def _test_wait_return_f(cls, barrier, queue): + res = barrier.wait() + queue.put(res) + + def test_wait_return(self): + """ + test the return value from barrier.wait + """ + queue = self.Queue() + self.run_threads(self._test_wait_return_f, (self.barrier, queue)) + results = [queue.get() for i in range(self.N)] + self.assertEqual(results.count(0), 1) + + @classmethod + def _test_action_f(cls, barrier, results): + barrier.wait() + if len(results) != 1: + raise RuntimeError + + def test_action(self): + """ + Test the 'action' callback + """ + results = self.DummyList() + barrier = self.Barrier(self.N, action=AppendTrue(results)) + self.run_threads(self._test_action_f, (barrier, results)) + self.assertEqual(len(results), 1) + + @classmethod + def _test_abort_f(cls, barrier, results1, results2): + try: + i = barrier.wait() + if i == cls.N//2: + raise RuntimeError + barrier.wait() + results1.append(True) + except threading.BrokenBarrierError: + results2.append(True) + except RuntimeError: + barrier.abort() + + def test_abort(self): + """ + Test that an abort will put the barrier in a broken state + """ + results1 = self.DummyList() + results2 = self.DummyList() + self.run_threads(self._test_abort_f, + (self.barrier, results1, results2)) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertTrue(self.barrier.broken) + + @classmethod + def _test_reset_f(cls, barrier, results1, results2, results3): + i = barrier.wait() + if i == cls.N//2: + # Wait until the other threads are all in the barrier. + while barrier.n_waiting < cls.N-1: + time.sleep(0.001) + barrier.reset() + else: + try: + barrier.wait() + results1.append(True) + except threading.BrokenBarrierError: + results2.append(True) + # Now, pass the barrier again + barrier.wait() + results3.append(True) + + def test_reset(self): + """ + Test that a 'reset' on a barrier frees the waiting threads + """ + results1 = self.DummyList() + results2 = self.DummyList() + results3 = self.DummyList() + self.run_threads(self._test_reset_f, + (self.barrier, results1, results2, results3)) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertEqual(len(results3), self.N) + + @classmethod + def _test_abort_and_reset_f(cls, barrier, barrier2, + results1, results2, results3): + try: + i = barrier.wait() + if i == cls.N//2: + raise RuntimeError + barrier.wait() + results1.append(True) + except threading.BrokenBarrierError: + results2.append(True) + except RuntimeError: + barrier.abort() + # Synchronize and reset the barrier. Must synchronize first so + # that everyone has left it when we reset, and after so that no + # one enters it before the reset. + if barrier2.wait() == cls.N//2: + barrier.reset() + barrier2.wait() + barrier.wait() + results3.append(True) + + def test_abort_and_reset(self): + """ + Test that a barrier can be reset after being broken. + """ + results1 = self.DummyList() + results2 = self.DummyList() + results3 = self.DummyList() + barrier2 = self.Barrier(self.N) + + self.run_threads(self._test_abort_and_reset_f, + (self.barrier, barrier2, results1, results2, results3)) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertEqual(len(results3), self.N) + + @classmethod + def _test_timeout_f(cls, barrier, results): + i = barrier.wait(20) + if i == cls.N//2: + # One thread is late! + time.sleep(4.0) + try: + barrier.wait(0.5) + except threading.BrokenBarrierError: + results.append(True) + + def test_timeout(self): + """ + Test wait(timeout) + """ + results = self.DummyList() + self.run_threads(self._test_timeout_f, (self.barrier, results)) + self.assertEqual(len(results), self.barrier.parties) + + @classmethod + def _test_default_timeout_f(cls, barrier, results): + i = barrier.wait(20) + if i == cls.N//2: + # One thread is later than the default timeout + time.sleep(4.0) + try: + barrier.wait() + except threading.BrokenBarrierError: + results.append(True) + + def test_default_timeout(self): + """ + Test the barrier's default timeout + """ + barrier = self.Barrier(self.N, timeout=1.0) + results = self.DummyList() + self.run_threads(self._test_default_timeout_f, (barrier, results)) + self.assertEqual(len(results), barrier.parties) + + def test_single_thread(self): + b = self.Barrier(1) + b.wait() + b.wait() + + @classmethod + def _test_thousand_f(cls, barrier, passes, conn, lock): + for i in range(passes): + barrier.wait() + with lock: + conn.send(i) + + def test_thousand(self): + if self.TYPE == 'manager': + return + passes = 1000 + lock = self.Lock() + conn, child_conn = self.Pipe(False) + for j in range(self.N): + p = self.Process(target=self._test_thousand_f, + args=(self.barrier, passes, child_conn, lock)) + p.start() + + for i in range(passes): + for j in range(self.N): + self.assertEqual(conn.recv(), i) + +# # # @@ -1485,6 +1820,11 @@ class _TestZZZNumberOfObjects(BaseTestCase): # run after all the other tests for the manager. It tests that # there have been no "reference leaks" for the manager's shared # objects. Note the comment in _TestPool.test_terminate(). + + # If some other test using ManagerMixin.manager fails, then the + # raised exception may keep alive a frame which holds a reference + # to a managed object. This will cause test_number_of_objects to + # also fail. ALLOWED_TYPES = ('manager',) def test_number_of_objects(self): @@ -1564,6 +1904,11 @@ class _TestMyManager(BaseTestCase): manager.shutdown() + # If the manager process exited cleanly then the exitcode + # will be zero. Otherwise (after a short timeout) + # terminate() is used, resulting in an exitcode of -SIGTERM. + self.assertEqual(manager._process.exitcode, 0) + # # Test of connecting to a remote server and using xmlrpclib for serialization # @@ -1923,7 +2268,7 @@ class _TestConnection(BaseTestCase): class _TestListener(BaseTestCase): - ALLOWED_TYPES = ('processes') + ALLOWED_TYPES = ('processes',) def test_multiple_bind(self): for family in self.connection.families: @@ -2505,10 +2850,12 @@ def create_test_cases(Mixin, type): result = {} glob = globals() Type = type.capitalize() + ALL_TYPES = {'processes', 'threads', 'manager'} for name in list(glob.keys()): if name.startswith('_Test'): base = glob[name] + assert set(base.ALLOWED_TYPES) <= ALL_TYPES, set(base.ALLOWED_TYPES) if type in base.ALLOWED_TYPES: newname = 'With' + Type + name[1:] class Temp(base, unittest.TestCase, Mixin): @@ -2527,7 +2874,7 @@ class ProcessesMixin(object): Process = multiprocessing.Process locals().update(get_attributes(multiprocessing, ( 'Queue', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', - 'Condition', 'Event', 'Value', 'Array', 'RawValue', + 'Condition', 'Event', 'Barrier', 'Value', 'Array', 'RawValue', 'RawArray', 'current_process', 'active_children', 'Pipe', 'connection', 'JoinableQueue', 'Pool' ))) @@ -2542,7 +2889,7 @@ class ManagerMixin(object): manager = object.__new__(multiprocessing.managers.SyncManager) locals().update(get_attributes(manager, ( 'Queue', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', - 'Condition', 'Event', 'Value', 'Array', 'list', 'dict', + 'Condition', 'Event', 'Barrier', 'Value', 'Array', 'list', 'dict', 'Namespace', 'JoinableQueue', 'Pool' ))) @@ -2555,7 +2902,7 @@ class ThreadsMixin(object): Process = multiprocessing.dummy.Process locals().update(get_attributes(multiprocessing.dummy, ( 'Queue', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', - 'Condition', 'Event', 'Value', 'Array', 'current_process', + 'Condition', 'Event', 'Barrier', 'Value', 'Array', 'current_process', 'active_children', 'Pipe', 'connection', 'dict', 'list', 'Namespace', 'JoinableQueue', 'Pool' ))) diff --git a/Lib/test/test_structseq.py b/Lib/test/test_structseq.py index d6c63b7..a89e955 100644 --- a/Lib/test/test_structseq.py +++ b/Lib/test/test_structseq.py @@ -78,8 +78,9 @@ class StructSeqTest(unittest.TestCase): def test_fields(self): t = time.gmtime() - self.assertEqual(len(t), t.n_fields) - self.assertEqual(t.n_fields, t.n_sequence_fields+t.n_unnamed_fields) + self.assertEqual(len(t), t.n_sequence_fields) + self.assertEqual(t.n_unnamed_fields, 0) + self.assertEqual(t.n_fields, time._STRUCT_TM_ITEMS) def test_constructor(self): t = time.struct_time diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 02f05c3..63e1453 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -31,15 +31,14 @@ class TimeTestCase(unittest.TestCase): time.time() info = time.get_clock_info('time') self.assertFalse(info.monotonic) - if sys.platform != 'win32': - self.assertTrue(info.adjusted) + self.assertTrue(info.adjustable) def test_clock(self): time.clock() info = time.get_clock_info('clock') self.assertTrue(info.monotonic) - self.assertFalse(info.adjusted) + self.assertFalse(info.adjustable) @unittest.skipUnless(hasattr(time, 'clock_gettime'), 'need time.clock_gettime()') @@ -371,10 +370,7 @@ class TimeTestCase(unittest.TestCase): info = time.get_clock_info('monotonic') self.assertTrue(info.monotonic) - if sys.platform == 'linux': - self.assertTrue(info.adjusted) - else: - self.assertFalse(info.adjusted) + self.assertFalse(info.adjustable) def test_perf_counter(self): time.perf_counter() @@ -390,7 +386,7 @@ class TimeTestCase(unittest.TestCase): info = time.get_clock_info('process_time') self.assertTrue(info.monotonic) - self.assertFalse(info.adjusted) + self.assertFalse(info.adjustable) @unittest.skipUnless(hasattr(time, 'monotonic'), 'need time.monotonic') @@ -441,7 +437,7 @@ class TimeTestCase(unittest.TestCase): # 0.0 < resolution <= 1.0 self.assertGreater(info.resolution, 0.0) self.assertLessEqual(info.resolution, 1.0) - self.assertIsInstance(info.adjusted, bool) + self.assertIsInstance(info.adjustable, bool) self.assertRaises(ValueError, time.get_clock_info, 'xxx') @@ -624,7 +620,58 @@ class TestPytime(unittest.TestCase): for invalid in self.invalid_values: self.assertRaises(OverflowError, pytime_object_to_timespec, invalid) + @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") + def test_localtime_timezone(self): + + # Get the localtime and examine it for the offset and zone. + lt = time.localtime() + self.assertTrue(hasattr(lt, "tm_gmtoff")) + self.assertTrue(hasattr(lt, "tm_zone")) + # See if the offset and zone are similar to the module + # attributes. + if lt.tm_gmtoff is None: + self.assertTrue(not hasattr(time, "timezone")) + else: + self.assertEqual(lt.tm_gmtoff, -[time.timezone, time.altzone][lt.tm_isdst]) + if lt.tm_zone is None: + self.assertTrue(not hasattr(time, "tzname")) + else: + self.assertEqual(lt.tm_zone, time.tzname[lt.tm_isdst]) + + # Try and make UNIX times from the localtime and a 9-tuple + # created from the localtime. Test to see that the times are + # the same. + t = time.mktime(lt); t9 = time.mktime(lt[:9]) + self.assertEqual(t, t9) + + # Make localtimes from the UNIX times and compare them to + # the original localtime, thus making a round trip. + new_lt = time.localtime(t); new_lt9 = time.localtime(t9) + self.assertEqual(new_lt, lt) + self.assertEqual(new_lt.tm_gmtoff, lt.tm_gmtoff) + self.assertEqual(new_lt.tm_zone, lt.tm_zone) + self.assertEqual(new_lt9, lt) + self.assertEqual(new_lt.tm_gmtoff, lt.tm_gmtoff) + self.assertEqual(new_lt9.tm_zone, lt.tm_zone) + + @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") + def test_strptime_timezone(self): + t = time.strptime("UTC", "%Z") + self.assertEqual(t.tm_zone, 'UTC') + t = time.strptime("+0500", "%z") + self.assertEqual(t.tm_gmtoff, 5 * 3600) + + @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") + def test_short_times(self): + + import pickle + + # Load a short time structure using pickle. + st = b"ctime\nstruct_time\np0\n((I2007\nI8\nI11\nI1\nI24\nI49\nI5\nI223\nI1\ntp1\n(dp2\ntp3\nRp4\n." + lt = pickle.loads(st) + self.assertIs(lt.tm_gmtoff, None) + self.assertIs(lt.tm_zone, None) def test_main(): support.run_unittest( diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 49a5633..24ecae5 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -23,7 +23,8 @@ import weakref from test import support from test.support import findfile, import_fresh_module, gc_collect -pyET = import_fresh_module('xml.etree.ElementTree', blocked=['_elementtree']) +pyET = None +ET = None SIMPLE_XMLFILE = findfile("simple.xml", subdir="xmltestdata") try: @@ -209,10 +210,8 @@ def interface(): These methods return an iterable. See bug 6472. - >>> check_method(element.iter("tag").__next__) >>> check_method(element.iterfind("tag").__next__) >>> check_method(element.iterfind("*").__next__) - >>> check_method(tree.iter("tag").__next__) >>> check_method(tree.iterfind("tag").__next__) >>> check_method(tree.iterfind("*").__next__) @@ -291,42 +290,6 @@ def cdata(): '<tag>hello</tag>' """ -# Only with Python implementation -def simplefind(): - """ - Test find methods using the elementpath fallback. - - >>> ElementTree = pyET - - >>> CurrentElementPath = ElementTree.ElementPath - >>> ElementTree.ElementPath = ElementTree._SimpleElementPath() - >>> elem = ElementTree.XML(SAMPLE_XML) - >>> elem.find("tag").tag - 'tag' - >>> ElementTree.ElementTree(elem).find("tag").tag - 'tag' - >>> elem.findtext("tag") - 'text' - >>> elem.findtext("tog") - >>> elem.findtext("tog", "default") - 'default' - >>> ElementTree.ElementTree(elem).findtext("tag") - 'text' - >>> summarize_list(elem.findall("tag")) - ['tag', 'tag'] - >>> summarize_list(elem.findall(".//tag")) - ['tag', 'tag', 'tag'] - - Path syntax doesn't work in this case. - - >>> elem.find("section/tag") - >>> elem.findtext("section/tag") - >>> summarize_list(elem.findall("section/tag")) - [] - - >>> ElementTree.ElementPath = CurrentElementPath - """ - def find(): """ Test find methods (including xpath syntax). @@ -1002,36 +965,6 @@ def methods(): '1 < 2\n' """ -def iterators(): - """ - Test iterators. - - >>> e = ET.XML("<html><body>this is a <i>paragraph</i>.</body>..</html>") - >>> summarize_list(e.iter()) - ['html', 'body', 'i'] - >>> summarize_list(e.find("body").iter()) - ['body', 'i'] - >>> summarize(next(e.iter())) - 'html' - >>> "".join(e.itertext()) - 'this is a paragraph...' - >>> "".join(e.find("body").itertext()) - 'this is a paragraph.' - >>> next(e.itertext()) - 'this is a ' - - Method iterparse should return an iterator. See bug 6472. - - >>> sourcefile = serialize(e, to_string=False) - >>> next(ET.iterparse(sourcefile)) # doctest: +ELLIPSIS - ('end', <Element 'i' at 0x...>) - - >>> tree = ET.ElementTree(None) - >>> tree.iter() - Traceback (most recent call last): - AttributeError: 'NoneType' object has no attribute 'iter' - """ - ENTITY_XML = """\ <!DOCTYPE points [ <!ENTITY % user-entities SYSTEM 'user-entities.xml'> @@ -1339,6 +1272,7 @@ XINCLUDE["default.xml"] = """\ </document> """.format(html.escape(SIMPLE_XMLFILE, True)) + def xinclude_loader(href, parse="xml", encoding=None): try: data = XINCLUDE[href] @@ -1411,22 +1345,6 @@ def xinclude(): >>> # print(serialize(document)) # C5 """ -def xinclude_default(): - """ - >>> from xml.etree import ElementInclude - - >>> document = xinclude_loader("default.xml") - >>> ElementInclude.include(document) - >>> print(serialize(document)) # default - <document> - <p>Example.</p> - <root> - <element key="value">text</element> - <element>text</element>tail - <empty-element /> - </root> - </document> - """ # # badly formatted xi:include tags @@ -1917,9 +1835,8 @@ class ElementTreeTest(unittest.TestCase): self.assertIsInstance(ET.QName, type) self.assertIsInstance(ET.ElementTree, type) self.assertIsInstance(ET.Element, type) - # XXX issue 14128 with C ElementTree - # self.assertIsInstance(ET.TreeBuilder, type) - # self.assertIsInstance(ET.XMLParser, type) + self.assertIsInstance(ET.TreeBuilder, type) + self.assertIsInstance(ET.XMLParser, type) def test_Element_subclass_trivial(self): class MyElement(ET.Element): @@ -1953,6 +1870,73 @@ class ElementTreeTest(unittest.TestCase): self.assertEqual(mye.newmethod(), 'joe') +class ElementIterTest(unittest.TestCase): + def _ilist(self, elem, tag=None): + return summarize_list(elem.iter(tag)) + + def test_basic(self): + doc = ET.XML("<html><body>this is a <i>paragraph</i>.</body>..</html>") + self.assertEqual(self._ilist(doc), ['html', 'body', 'i']) + self.assertEqual(self._ilist(doc.find('body')), ['body', 'i']) + self.assertEqual(next(doc.iter()).tag, 'html') + self.assertEqual(''.join(doc.itertext()), 'this is a paragraph...') + self.assertEqual(''.join(doc.find('body').itertext()), + 'this is a paragraph.') + self.assertEqual(next(doc.itertext()), 'this is a ') + + # iterparse should return an iterator + sourcefile = serialize(doc, to_string=False) + self.assertEqual(next(ET.iterparse(sourcefile))[0], 'end') + + tree = ET.ElementTree(None) + self.assertRaises(AttributeError, tree.iter) + + def test_corners(self): + # single root, no subelements + a = ET.Element('a') + self.assertEqual(self._ilist(a), ['a']) + + # one child + b = ET.SubElement(a, 'b') + self.assertEqual(self._ilist(a), ['a', 'b']) + + # one child and one grandchild + c = ET.SubElement(b, 'c') + self.assertEqual(self._ilist(a), ['a', 'b', 'c']) + + # two children, only first with grandchild + d = ET.SubElement(a, 'd') + self.assertEqual(self._ilist(a), ['a', 'b', 'c', 'd']) + + # replace first child by second + a[0] = a[1] + del a[1] + self.assertEqual(self._ilist(a), ['a', 'd']) + + def test_iter_by_tag(self): + doc = ET.XML(''' + <document> + <house> + <room>bedroom1</room> + <room>bedroom2</room> + </house> + <shed>nothing here + </shed> + <house> + <room>bedroom8</room> + </house> + </document>''') + + self.assertEqual(self._ilist(doc, 'room'), ['room'] * 3) + self.assertEqual(self._ilist(doc, 'house'), ['house'] * 2) + + # make sure both tag=None and tag='*' return all tags + all_tags = ['document', 'house', 'room', 'room', + 'shed', 'house', 'room'] + self.assertEqual(self._ilist(doc), all_tags) + self.assertEqual(self._ilist(doc, '*'), all_tags) + + class TreeBuilderTest(unittest.TestCase): sample1 = ('<!DOCTYPE html PUBLIC' ' "-//W3C//DTD XHTML 1.0 Transitional//EN"' @@ -2027,6 +2011,23 @@ class TreeBuilderTest(unittest.TestCase): 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd')) +@unittest.skip('Unstable due to module monkeypatching') +class XincludeTest(unittest.TestCase): + def test_xinclude_default(self): + from xml.etree import ElementInclude + doc = xinclude_loader('default.xml') + ElementInclude.include(doc) + s = serialize(doc) + self.assertEqual(s.strip(), '''<document> + <p>Example.</p> + <root> + <element key="value">text</element> + <element>text</element>tail + <empty-element /> +</root> +</document>''') + + class XMLParserTest(unittest.TestCase): sample1 = '<file><line>22</line></file>' sample2 = ('<!DOCTYPE html PUBLIC' @@ -2073,13 +2074,6 @@ class XMLParserTest(unittest.TestCase): 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd')) -class NoAcceleratorTest(unittest.TestCase): - # Test that the C accelerator was not imported for pyET - def test_correct_import_pyET(self): - self.assertEqual(pyET.Element.__module__, 'xml.etree.ElementTree') - self.assertEqual(pyET.SubElement.__module__, 'xml.etree.ElementTree') - - class NamespaceParseTest(unittest.TestCase): def test_find_with_namespace(self): nsmap = {'h': 'hello', 'f': 'foo'} @@ -2090,7 +2084,6 @@ class NamespaceParseTest(unittest.TestCase): self.assertEqual(len(doc.findall('.//{foo}name', nsmap)), 1) - class ElementSlicingTest(unittest.TestCase): def _elem_tags(self, elemlist): return [e.tag for e in elemlist] @@ -2232,6 +2225,14 @@ class KeywordArgsTest(unittest.TestCase): with self.assertRaisesRegex(TypeError, 'must be dict, not str'): ET.Element('a', attrib="I'm not a dict") +# -------------------------------------------------------------------- + +@unittest.skipUnless(pyET, 'only for the Python version') +class NoAcceleratorTest(unittest.TestCase): + # Test that the C accelerator was not imported for pyET + def test_correct_import_pyET(self): + self.assertEqual(pyET.Element.__module__, 'xml.etree.ElementTree') + self.assertEqual(pyET.SubElement.__module__, 'xml.etree.ElementTree') # -------------------------------------------------------------------- @@ -2276,31 +2277,42 @@ class CleanContext(object): self.checkwarnings.__exit__(*args) -def test_main(module=pyET): - from test import test_xml_etree +def test_main(module=None): + # When invoked without a module, runs the Python ET tests by loading pyET. + # Otherwise, uses the given module as the ET. + if module is None: + global pyET + pyET = import_fresh_module('xml.etree.ElementTree', + blocked=['_elementtree']) + module = pyET - # The same doctests are used for both the Python and the C implementations - test_xml_etree.ET = module + global ET + ET = module test_classes = [ ElementSlicingTest, BasicElementTest, StringIOTest, ParseErrorTest, + XincludeTest, ElementTreeTest, - NamespaceParseTest, + ElementIterTest, TreeBuilderTest, - XMLParserTest, - KeywordArgsTest] - if module is pyET: - # Run the tests specific to the Python implementation - test_classes += [NoAcceleratorTest] + ] + + # These tests will only run for the pure-Python version that doesn't import + # _elementtree. We can't use skipUnless here, because pyET is filled in only + # after the module is loaded. + if pyET: + test_classes.extend([ + NoAcceleratorTest, + ]) support.run_unittest(*test_classes) # XXX the C module should give the same warnings as the Python module with CleanContext(quiet=(module is not pyET)): - support.run_doctest(test_xml_etree, verbosity=True) + support.run_doctest(sys.modules[__name__], verbosity=True) if __name__ == '__main__': test_main() diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py index 10416d2..142a22f 100644 --- a/Lib/test/test_xml_etree_c.py +++ b/Lib/test/test_xml_etree_c.py @@ -8,31 +8,6 @@ cET = import_fresh_module('xml.etree.ElementTree', fresh=['_elementtree']) cET_alias = import_fresh_module('xml.etree.cElementTree', fresh=['_elementtree', 'xml.etree']) -# cElementTree specific tests - -def sanity(): - r""" - Import sanity. - - Issue #6697. - - >>> cElementTree = cET - >>> e = cElementTree.Element('a') - >>> getattr(e, '\uD800') # doctest: +ELLIPSIS - Traceback (most recent call last): - ... - UnicodeEncodeError: ... - - >>> p = cElementTree.XMLParser() - >>> p.version.split()[0] - 'Expat' - >>> getattr(p, '\uD800') - Traceback (most recent call last): - ... - AttributeError: 'XMLParser' object has no attribute '\ud800' - """ - - class MiscTests(unittest.TestCase): # Issue #8651. @support.bigmemtest(size=support._2G + 100, memuse=1) @@ -46,6 +21,7 @@ class MiscTests(unittest.TestCase): finally: data = None + @unittest.skipUnless(cET, 'requires _elementtree') class TestAliasWorking(unittest.TestCase): # Test that the cET alias module is alive @@ -53,6 +29,7 @@ class TestAliasWorking(unittest.TestCase): e = cET_alias.Element('foo') self.assertEqual(e.tag, 'foo') + @unittest.skipUnless(cET, 'requires _elementtree') class TestAcceleratorImported(unittest.TestCase): # Test that the C accelerator was imported, as expected @@ -67,7 +44,6 @@ def test_main(): from test import test_xml_etree, test_xml_etree_c # Run the tests specific to the C implementation - support.run_doctest(test_xml_etree_c, verbosity=True) support.run_unittest( MiscTests, TestAliasWorking, diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index e068fc2..d30a83c 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -101,32 +101,8 @@ import sys import re import warnings -class _SimpleElementPath: - # emulate pre-1.2 find/findtext/findall behaviour - def find(self, element, tag, namespaces=None): - for elem in element: - if elem.tag == tag: - return elem - return None - def findtext(self, element, tag, default=None, namespaces=None): - elem = self.find(element, tag) - if elem is None: - return default - return elem.text or "" - def iterfind(self, element, tag, namespaces=None): - if tag[:3] == ".//": - for elem in element.iter(tag[3:]): - yield elem - for elem in element: - if elem.tag == tag: - yield elem - def findall(self, element, tag, namespaces=None): - return list(self.iterfind(element, tag, namespaces)) +from . import ElementPath -try: - from . import ElementPath -except ImportError: - ElementPath = _SimpleElementPath() ## # Parser error. This is a subclass of <b>SyntaxError</b>. @@ -916,11 +892,7 @@ def _namespaces(elem, default_namespace=None): _raise_serialization_error(qname) # populate qname and namespaces table - try: - iterate = elem.iter - except AttributeError: - iterate = elem.getiterator # cET compatibility - for elem in iterate(): + for elem in elem.iter(): tag = elem.tag if isinstance(tag, QName): if tag.text not in qnames: @@ -10,6 +10,9 @@ What's New in Python 3.3.0 Beta 1? Core and Builtins ----------------- +- Issue #15026: utf-16 encoding is now significantly faster (up to 10x). + Patch by Serhiy Storchaka. + - Issue #11022: open() and io.TextIOWrapper are now calling locale.getpreferredencoding(False) instead of locale.getpreferredencoding() in text mode if the encoding is not specified. Don't change temporary the @@ -21,6 +24,32 @@ Core and Builtins Library ------- +- Issue #15036: Allow removing or changing multiple items in + single-file mailboxes (mbox, MMDF, Babyl) flushing the mailbox + between the changes. + +- Issue #14059: Implement multiprocessing.Barrier. + +- Issue #15061: The inappropriately named hmac.secure_compare has been + renamed to hmac.compare_digest, restricted to operating on bytes inputs + only and had its documentation updated to more accurately reflect both its + intent and its limitations + +- Issue #13841: Make child processes exit using sys.exit() on Windows. + +- Issue #14936: curses_panel was converted to PEP 3121 and PEP 384 API. + Patch by Robin Schreiber. + +- Issue #1667546: On platforms supporting tm_zone and tm_gmtoff fields + in struct tm, time.struct_time objects returned by time.gmtime(), + time.localtime() and time.strptime() functions now have tm_zone and + tm_gmtoff attributes. Original patch by Paul Boddie. + +- Rename adjusted attribute to adjustable in time.get_clock_info() result. + +- Issue #3518: Remove references to non-existent BaseManager.from_address() + method. + - Issue #13857: Added textwrap.indent() function (initial patch by Ezra Berch) diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index 8561a2e..1c7d084 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -16,8 +16,37 @@ static char *PyCursesVersion = "2.1"; #include <panel.h> -static PyObject *PyCursesError; +typedef struct { + PyObject *PyCursesError; + PyObject *PyCursesPanel_Type; +} _curses_panelstate; + +#define _curses_panelstate(o) ((_curses_panelstate *)PyModule_GetState(o)) + +static int +_curses_panel_clear(PyObject *m) +{ + Py_CLEAR(_curses_panelstate(m)->PyCursesError); + return 0; +} + +static int +_curses_panel_traverse(PyObject *m, visitproc visit, void *arg) +{ + Py_VISIT(_curses_panelstate(m)->PyCursesError); + return 0; +} + +static void +_curses_panel_free(void *m) +{ + _curses_panel_clear((PyObject *) m); +} +static struct PyModuleDef _curses_panelmodule; + +#define _curses_panelstate_global \ +((_curses_panelstate *) PyModule_GetState(PyState_FindModule(&_curses_panelmodule))) /* Utility Functions */ @@ -34,9 +63,9 @@ PyCursesCheckERR(int code, char *fname) return Py_None; } else { if (fname == NULL) { - PyErr_SetString(PyCursesError, catchall_ERR); + PyErr_SetString(_curses_panelstate_global->PyCursesError, catchall_ERR); } else { - PyErr_Format(PyCursesError, "%s() returned ERR", fname); + PyErr_Format(_curses_panelstate_global->PyCursesError, "%s() returned ERR", fname); } return NULL; } @@ -54,9 +83,8 @@ typedef struct { PyCursesWindowObject *wo; /* for reference counts */ } PyCursesPanelObject; -PyTypeObject PyCursesPanel_Type; - -#define PyCursesPanel_Check(v) (Py_TYPE(v) == &PyCursesPanel_Type) +#define PyCursesPanel_Check(v) \ + (Py_TYPE(v) == _curses_panelstate_global->PyCursesPanel_Type) /* Some helper functions. The problem is that there's always a window associated with a panel. To ensure that Python's GC doesn't pull @@ -175,7 +203,8 @@ PyCursesPanel_New(PANEL *pan, PyCursesWindowObject *wo) { PyCursesPanelObject *po; - po = PyObject_NEW(PyCursesPanelObject, &PyCursesPanel_Type); + po = PyObject_NEW(PyCursesPanelObject, + (PyTypeObject *)(_curses_panelstate_global)->PyCursesPanel_Type); if (po == NULL) return NULL; po->pan = pan; if (insert_lop(po) < 0) { @@ -280,7 +309,7 @@ PyCursesPanel_replace_panel(PyCursesPanelObject *self, PyObject *args) rtn = replace_panel(self->pan, temp->win); if (rtn == ERR) { - PyErr_SetString(PyCursesError, "replace_panel() returned ERR"); + PyErr_SetString(_curses_panelstate_global->PyCursesError, "replace_panel() returned ERR"); return NULL; } Py_DECREF(po->wo); @@ -305,7 +334,7 @@ PyCursesPanel_userptr(PyCursesPanelObject *self) PyCursesInitialised; obj = (PyObject *) panel_userptr(self->pan); if (obj == NULL) { - PyErr_SetString(PyCursesError, "no userptr set"); + PyErr_SetString(_curses_panelstate_global->PyCursesError, "no userptr set"); return NULL; } @@ -334,36 +363,18 @@ static PyMethodDef PyCursesPanel_Methods[] = { /* -------------------------------------------------------*/ -PyTypeObject PyCursesPanel_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_curses_panel.curses panel", /*tp_name*/ - sizeof(PyCursesPanelObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - /* methods */ - (destructor)PyCursesPanel_Dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_reserved*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - PyCursesPanel_Methods, /*tp_methods*/ +static PyType_Slot PyCursesPanel_Type_slots[] = { + {Py_tp_dealloc, PyCursesPanel_Dealloc}, + {Py_tp_methods, PyCursesPanel_Methods}, + {0, 0}, +}; + +static PyType_Spec PyCursesPanel_Type_spec = { + "_curses_panel.curses panel", + sizeof(PyCursesPanelObject), + 0, + Py_TPFLAGS_DEFAULT, + PyCursesPanel_Type_slots }; /* Wrapper for panel_above(NULL). This function returns the bottom @@ -405,7 +416,7 @@ PyCurses_new_panel(PyObject *self, PyObject *args) return NULL; pan = new_panel(win->win); if (pan == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + PyErr_SetString(_curses_panelstate_global->PyCursesError, catchall_NULL); return NULL; } return (PyObject *)PyCursesPanel_New(pan, win); @@ -467,12 +478,12 @@ static struct PyModuleDef _curses_panelmodule = { PyModuleDef_HEAD_INIT, "_curses_panel", NULL, - -1, + sizeof(_curses_panelstate), PyCurses_methods, NULL, - NULL, - NULL, - NULL + _curses_panel_traverse, + _curses_panel_clear, + _curses_panel_free }; PyMODINIT_FUNC @@ -480,21 +491,23 @@ PyInit__curses_panel(void) { PyObject *m, *d, *v; - /* Initialize object type */ - if (PyType_Ready(&PyCursesPanel_Type) < 0) - return NULL; - - import_curses(); - /* Create the module and add the functions */ m = PyModule_Create(&_curses_panelmodule); if (m == NULL) - return NULL; + goto fail; d = PyModule_GetDict(m); + /* Initialize object type */ + _curses_panelstate(m)->PyCursesPanel_Type = \ + PyType_FromSpec(&PyCursesPanel_Type_spec); + if (_curses_panelstate(m)->PyCursesPanel_Type == NULL) + goto fail; + + import_curses(); + /* For exception _curses_panel.error */ - PyCursesError = PyErr_NewException("_curses_panel.error", NULL, NULL); - PyDict_SetItemString(d, "error", PyCursesError); + _curses_panelstate(m)->PyCursesError = PyErr_NewException("_curses_panel.error", NULL, NULL); + PyDict_SetItemString(d, "error", _curses_panelstate(m)->PyCursesError); /* Make the version available */ v = PyUnicode_FromString(PyCursesVersion); @@ -502,4 +515,7 @@ PyInit__curses_panel(void) PyDict_SetItemString(d, "__version__", v); Py_DECREF(v); return m; + fail: + Py_XDECREF(m); + return NULL; } diff --git a/Modules/_decimal/libmpdec/mpdecimal.c b/Modules/_decimal/libmpdec/mpdecimal.c index ff6d867..262e834 100644 --- a/Modules/_decimal/libmpdec/mpdecimal.c +++ b/Modules/_decimal/libmpdec/mpdecimal.c @@ -107,8 +107,9 @@ static inline void _mpd_qmul(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); static void _mpd_base_ndivmod(mpd_t *q, mpd_t *r, const mpd_t *a, const mpd_t *b, uint32_t *status); -static inline void _mpd_qpow_uint(mpd_t *result, mpd_t *base, mpd_uint_t exp, - uint8_t resultsign, const mpd_context_t *ctx, uint32_t *status); +static inline void _mpd_qpow_uint(mpd_t *result, const mpd_t *base, + mpd_uint_t exp, uint8_t resultsign, + const mpd_context_t *ctx, uint32_t *status); mpd_uint_t mpd_qsshiftr(mpd_t *result, const mpd_t *a, mpd_ssize_t n); @@ -5841,12 +5842,12 @@ mpd_qnext_toward(mpd_t *result, const mpd_t *a, const mpd_t *b, } /* - * Internal function: Integer power with mpd_uint_t exponent, base is modified! - * Function can fail with MPD_Malloc_error. + * Internal function: Integer power with mpd_uint_t exponent. The function + * can fail with MPD_Malloc_error. */ static inline void -_mpd_qpow_uint(mpd_t *result, mpd_t *base, mpd_uint_t exp, uint8_t resultsign, - const mpd_context_t *ctx, uint32_t *status) +_mpd_qpow_uint(mpd_t *result, const mpd_t *base, mpd_uint_t exp, + uint8_t resultsign, const mpd_context_t *ctx, uint32_t *status) { uint32_t workstatus = 0; mpd_uint_t n; @@ -5866,7 +5867,8 @@ _mpd_qpow_uint(mpd_t *result, mpd_t *base, mpd_uint_t exp, uint8_t resultsign, if (exp & n) { mpd_qmul(result, result, base, ctx, &workstatus); } - if (workstatus & (MPD_Overflow|MPD_Clamped)) { + if (mpd_isspecial(result) || + (mpd_iszerocoeff(result) && (workstatus & MPD_Clamped))) { break; } } diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 103e778..cb84048 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -103,8 +103,6 @@ do { memory -= size; printf("%8d - %s\n", memory, comment); } while (0) /* glue functions (see the init function for details) */ static PyObject* elementtree_parseerror_obj; static PyObject* elementtree_deepcopy_obj; -static PyObject* elementtree_iter_obj; -static PyObject* elementtree_itertext_obj; static PyObject* elementpath_obj; /* helpers */ @@ -1109,67 +1107,32 @@ element_getchildren(ElementObject* self, PyObject* args) return list; } -static PyObject* -element_iter(ElementObject* self, PyObject* args) -{ - PyObject* result; - PyObject* tag = Py_None; - if (!PyArg_ParseTuple(args, "|O:iter", &tag)) - return NULL; +static PyObject * +create_elementiter(ElementObject *self, PyObject *tag, int gettext); - if (!elementtree_iter_obj) { - PyErr_SetString( - PyExc_RuntimeError, - "iter helper not found" - ); - return NULL; - } - args = PyTuple_New(2); - if (!args) +static PyObject * +element_iter(ElementObject *self, PyObject *args) +{ + PyObject* tag = Py_None; + if (!PyArg_ParseTuple(args, "|O:iter", &tag)) return NULL; - Py_INCREF(self); PyTuple_SET_ITEM(args, 0, (PyObject*) self); - Py_INCREF(tag); PyTuple_SET_ITEM(args, 1, (PyObject*) tag); - - result = PyObject_CallObject(elementtree_iter_obj, args); - - Py_DECREF(args); - - return result; + return create_elementiter(self, tag, 0); } static PyObject* element_itertext(ElementObject* self, PyObject* args) { - PyObject* result; - if (!PyArg_ParseTuple(args, ":itertext")) return NULL; - if (!elementtree_itertext_obj) { - PyErr_SetString( - PyExc_RuntimeError, - "itertext helper not found" - ); - return NULL; - } - - args = PyTuple_New(1); - if (!args) - return NULL; - - Py_INCREF(self); PyTuple_SET_ITEM(args, 0, (PyObject*) self); - - result = PyObject_CallObject(elementtree_itertext_obj, args); - - Py_DECREF(args); - - return result; + return create_elementiter(self, Py_None, 1); } + static PyObject* element_getitem(PyObject* self_, Py_ssize_t index) { @@ -1790,6 +1753,269 @@ static PyTypeObject Element_Type = { 0, /* tp_free */ }; +/******************************* Element iterator ****************************/ + +/* ElementIterObject represents the iteration state over an XML element in + * pre-order traversal. To keep track of which sub-element should be returned + * next, a stack of parents is maintained. This is a standard stack-based + * iterative pre-order traversal of a tree. + * The stack is managed using a single-linked list starting at parent_stack. + * Each stack node contains the saved parent to which we should return after + * the current one is exhausted, and the next child to examine in that parent. + */ +typedef struct ParentLocator_t { + ElementObject *parent; + Py_ssize_t child_index; + struct ParentLocator_t *next; +} ParentLocator; + +typedef struct { + PyObject_HEAD + ParentLocator *parent_stack; + ElementObject *root_element; + PyObject *sought_tag; + int root_done; + int gettext; +} ElementIterObject; + + +static void +elementiter_dealloc(ElementIterObject *it) +{ + ParentLocator *p = it->parent_stack; + while (p) { + ParentLocator *temp = p; + Py_XDECREF(p->parent); + p = p->next; + PyObject_Free(temp); + } + + Py_XDECREF(it->sought_tag); + Py_XDECREF(it->root_element); + + PyObject_GC_UnTrack(it); + PyObject_GC_Del(it); +} + +static int +elementiter_traverse(ElementIterObject *it, visitproc visit, void *arg) +{ + ParentLocator *p = it->parent_stack; + while (p) { + Py_VISIT(p->parent); + p = p->next; + } + + Py_VISIT(it->root_element); + Py_VISIT(it->sought_tag); + return 0; +} + +/* Helper function for elementiter_next. Add a new parent to the parent stack. + */ +static ParentLocator * +parent_stack_push_new(ParentLocator *stack, ElementObject *parent) +{ + ParentLocator *new_node = PyObject_Malloc(sizeof(ParentLocator)); + if (new_node) { + new_node->parent = parent; + Py_INCREF(parent); + new_node->child_index = 0; + new_node->next = stack; + } + return new_node; +} + +static PyObject * +elementiter_next(ElementIterObject *it) +{ + /* Sub-element iterator. + * + * A short note on gettext: this function serves both the iter() and + * itertext() methods to avoid code duplication. However, there are a few + * small differences in the way these iterations work. Namely: + * - itertext() only yields text from nodes that have it, and continues + * iterating when a node doesn't have text (so it doesn't return any + * node like iter()) + * - itertext() also has to handle tail, after finishing with all the + * children of a node. + */ + ElementObject *cur_parent; + Py_ssize_t child_index; + + while (1) { + /* Handle the case reached in the beginning and end of iteration, where + * the parent stack is empty. The root_done flag gives us indication + * whether we've just started iterating (so root_done is 0), in which + * case the root is returned. If root_done is 1 and we're here, the + * iterator is exhausted. + */ + if (!it->parent_stack->parent) { + if (it->root_done) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } else { + it->parent_stack = parent_stack_push_new(it->parent_stack, + it->root_element); + if (!it->parent_stack) { + PyErr_NoMemory(); + return NULL; + } + + it->root_done = 1; + if (it->sought_tag == Py_None || + PyObject_RichCompareBool(it->root_element->tag, + it->sought_tag, Py_EQ) == 1) { + if (it->gettext) { + PyObject *text = JOIN_OBJ(it->root_element->text); + if (PyObject_IsTrue(text)) { + Py_INCREF(text); + return text; + } + } else { + Py_INCREF(it->root_element); + return (PyObject *)it->root_element; + } + } + } + } + + /* See if there are children left to traverse in the current parent. If + * yes, visit the next child. If not, pop the stack and try again. + */ + cur_parent = it->parent_stack->parent; + child_index = it->parent_stack->child_index; + if (cur_parent->extra && child_index < cur_parent->extra->length) { + ElementObject *child = (ElementObject *) + cur_parent->extra->children[child_index]; + it->parent_stack->child_index++; + it->parent_stack = parent_stack_push_new(it->parent_stack, + child); + if (!it->parent_stack) { + PyErr_NoMemory(); + return NULL; + } + + if (it->gettext) { + PyObject *text = JOIN_OBJ(child->text); + if (PyObject_IsTrue(text)) { + Py_INCREF(text); + return text; + } + } else if (it->sought_tag == Py_None || + PyObject_RichCompareBool(child->tag, + it->sought_tag, Py_EQ) == 1) { + Py_INCREF(child); + return (PyObject *)child; + } + else + continue; + } + else { + PyObject *tail = it->gettext ? JOIN_OBJ(cur_parent->tail) : Py_None; + ParentLocator *next = it->parent_stack->next; + Py_XDECREF(it->parent_stack->parent); + PyObject_Free(it->parent_stack); + it->parent_stack = next; + + /* Note that extra condition on it->parent_stack->parent here; + * this is because itertext() is supposed to only return *inner* + * text, not text following the element it began iteration with. + */ + if (it->parent_stack->parent && PyObject_IsTrue(tail)) { + Py_INCREF(tail); + return tail; + } + } + } + + return NULL; +} + + +static PyTypeObject ElementIter_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_elementtree._element_iterator", /* tp_name */ + sizeof(ElementIterObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)elementiter_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + 0, /* tp_doc */ + (traverseproc)elementiter_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + (iternextfunc)elementiter_next, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; + + +static PyObject * +create_elementiter(ElementObject *self, PyObject *tag, int gettext) +{ + ElementIterObject *it; + PyObject *star = NULL; + + it = PyObject_GC_New(ElementIterObject, &ElementIter_Type); + if (!it) + return NULL; + if (!(it->parent_stack = PyObject_Malloc(sizeof(ParentLocator)))) { + PyObject_GC_Del(it); + return NULL; + } + + it->parent_stack->parent = NULL; + it->parent_stack->child_index = 0; + it->parent_stack->next = NULL; + + if (PyUnicode_Check(tag)) + star = PyUnicode_FromString("*"); + else if (PyBytes_Check(tag)) + star = PyBytes_FromString("*"); + + if (star && PyObject_RichCompareBool(tag, star, Py_EQ) == 1) + tag = Py_None; + + Py_XDECREF(star); + it->sought_tag = tag; + it->root_done = 0; + it->gettext = gettext; + it->root_element = self; + + Py_INCREF(self); + Py_INCREF(tag); + + PyObject_GC_Track(it); + return (PyObject *)it; +} + + /* ==================================================================== */ /* the tree builder type */ @@ -3238,8 +3464,7 @@ static struct PyModuleDef _elementtreemodule = { PyMODINIT_FUNC PyInit__elementtree(void) { - PyObject *m, *g, *temp; - char* bootstrap; + PyObject *m, *temp; /* Initialize object types */ if (PyType_Ready(&TreeBuilder_Type) < 0) @@ -3255,44 +3480,6 @@ PyInit__elementtree(void) if (!m) return NULL; - /* The code below requires that the module gets already added - to sys.modules. */ - PyDict_SetItemString(PyImport_GetModuleDict(), - _elementtreemodule.m_name, - m); - - /* python glue code */ - - g = PyDict_New(); - if (!g) - return NULL; - - PyDict_SetItemString(g, "__builtins__", PyEval_GetBuiltins()); - - bootstrap = ( - "def iter(node, tag=None):\n" /* helper */ - " if tag == '*':\n" - " tag = None\n" - " if tag is None or node.tag == tag:\n" - " yield node\n" - " for node in node:\n" - " for node in iter(node, tag):\n" - " yield node\n" - - "def itertext(node):\n" /* helper */ - " if node.text:\n" - " yield node.text\n" - " for e in node:\n" - " for s in e.itertext():\n" - " yield s\n" - " if e.tail:\n" - " yield e.tail\n" - - ); - - if (!PyRun_String(bootstrap, Py_file_input, g, NULL)) - return NULL; - if (!(temp = PyImport_ImportModule("copy"))) return NULL; elementtree_deepcopy_obj = PyObject_GetAttrString(temp, "deepcopy"); @@ -3301,9 +3488,6 @@ PyInit__elementtree(void) if (!(elementpath_obj = PyImport_ImportModule("xml.etree.ElementPath"))) return NULL; - elementtree_iter_obj = PyDict_GetItemString(g, "iter"); - elementtree_itertext_obj = PyDict_GetItemString(g, "itertext"); - /* link against pyexpat */ expat_capi = PyCapsule_Import(PyExpat_CAPSULE_NAME, 0); if (expat_capi) { diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 5961ac9..161407d 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -96,7 +96,7 @@ floatclock(_Py_clock_info_t *info) info->implementation = "clock()"; info->resolution = 1.0 / (double)CLOCKS_PER_SEC; info->monotonic = 1; - info->adjusted = 0; + info->adjustable = 0; } return PyFloat_FromDouble((double)value / CLOCKS_PER_SEC); } @@ -132,7 +132,7 @@ win_perf_counter(_Py_clock_info_t *info, PyObject **result) info->implementation = "QueryPerformanceCounter()"; info->resolution = 1.0 / (double)cpu_frequency; info->monotonic = 1; - info->adjusted = 0; + info->adjustable = 0; } *result = PyFloat_FromDouble(diff / (double)cpu_frequency); return 0; @@ -275,6 +275,10 @@ static PyStructSequence_Field struct_time_type_fields[] = { {"tm_wday", "day of week, range [0, 6], Monday is 0"}, {"tm_yday", "day of year, range [1, 366]"}, {"tm_isdst", "1 if summer time is in effect, 0 if not, and -1 if unknown"}, +#ifdef HAVE_STRUCT_TM_TM_ZONE + {"tm_zone", "abbreviation of timezone name"}, + {"tm_gmtoff", "offset from UTC in seconds"}, +#endif /* HAVE_STRUCT_TM_TM_ZONE */ {0} }; @@ -294,6 +298,7 @@ static PyStructSequence_Desc struct_time_type_desc = { static int initialized; static PyTypeObject StructTimeType; + static PyObject * tmtotuple(struct tm *p) { @@ -312,6 +317,11 @@ tmtotuple(struct tm *p) SET(6, (p->tm_wday + 6) % 7); /* Want Monday == 0 */ SET(7, p->tm_yday + 1); /* Want January, 1 == 1 */ SET(8, p->tm_isdst); +#ifdef HAVE_STRUCT_TM_TM_ZONE + PyStructSequence_SET_ITEM(v, 9, + PyUnicode_DecodeLocale(p->tm_zone, "surrogateescape")); + SET(10, p->tm_gmtoff); +#endif /* HAVE_STRUCT_TM_TM_ZONE */ #undef SET if (PyErr_Occurred()) { Py_XDECREF(v); @@ -371,7 +381,10 @@ PyDoc_STRVAR(gmtime_doc, tm_sec, tm_wday, tm_yday, tm_isdst)\n\ \n\ Convert seconds since the Epoch to a time tuple expressing UTC (a.k.a.\n\ -GMT). When 'seconds' is not passed in, convert the current time instead."); +GMT). When 'seconds' is not passed in, convert the current time instead.\n\ +\n\ +If the platform supports the tm_gmtoff and tm_zone, they are available as\n\ +attributes only."); static int pylocaltime(time_t *timep, struct tm *result) @@ -401,7 +414,7 @@ time_localtime(PyObject *self, PyObject *args) if (!parse_time_t_args(args, "|O:localtime", &when)) return NULL; - if (pylocaltime(&when, &buf) == 1) + if (pylocaltime(&when, &buf) == -1) return NULL; return tmtotuple(&buf); } @@ -438,6 +451,17 @@ gettmarg(PyObject *args, struct tm *p) p->tm_mon--; p->tm_wday = (p->tm_wday + 1) % 7; p->tm_yday--; +#ifdef HAVE_STRUCT_TM_TM_ZONE + if (Py_TYPE(args) == &StructTimeType) { + PyObject *item; + item = PyTuple_GET_ITEM(args, 9); + p->tm_zone = item == Py_None ? NULL : _PyUnicode_AsString(item); + item = PyTuple_GET_ITEM(args, 10); + p->tm_gmtoff = item == Py_None ? 0 : PyLong_AsLong(item); + if (PyErr_Occurred()) + return 0; + } +#endif /* HAVE_STRUCT_TM_TM_ZONE */ return 1; } @@ -778,7 +802,10 @@ time_mktime(PyObject *self, PyObject *tup) PyDoc_STRVAR(mktime_doc, "mktime(tuple) -> floating point number\n\ \n\ -Convert a time tuple in local time to seconds since the Epoch."); +Convert a time tuple in local time to seconds since the Epoch.\n\ +Note that mktime(gmtime(0)) will not generally return zero for most\n\ +time zones; instead the returned value will either be equal to that\n\ +of the timezone or altzone attributes on the time module."); #endif /* HAVE_MKTIME */ #ifdef HAVE_WORKING_TZSET @@ -882,7 +909,7 @@ pymonotonic(_Py_clock_info_t *info) return NULL; } info->resolution = timeIncrement * 1e-7; - info->adjusted = 0; + info->adjustable = 0; } return PyFloat_FromDouble(result); @@ -903,7 +930,7 @@ pymonotonic(_Py_clock_info_t *info) info->implementation = "mach_absolute_time()"; info->resolution = (double)timebase.numer / timebase.denom * 1e-9; info->monotonic = 1; - info->adjusted = 0; + info->adjustable = 0; } return PyFloat_FromDouble(secs); @@ -926,13 +953,7 @@ pymonotonic(_Py_clock_info_t *info) struct timespec res; info->monotonic = 1; info->implementation = function; -#if (defined(linux) || defined(__linux) || defined(__linux__)) \ - && !defined(CLOCK_HIGHRES) - /* CLOCK_MONOTONIC is adjusted on Linux */ - info->adjusted = 1; -#else - info->adjusted = 0; -#endif + info->adjustable = 0; if (clock_getres(clk_id, &res) == 0) info->resolution = res.tv_sec + res.tv_nsec * 1e-9; else @@ -1024,7 +1045,7 @@ py_process_time(_Py_clock_info_t *info) info->implementation = "GetProcessTimes()"; info->resolution = 1e-7; info->monotonic = 1; - info->adjusted = 0; + info->adjustable = 0; } return PyFloat_FromDouble(total * 1e-7); #else @@ -1053,7 +1074,7 @@ py_process_time(_Py_clock_info_t *info) struct timespec res; info->implementation = function; info->monotonic = 1; - info->adjusted = 0; + info->adjustable = 0; if (clock_getres(clk_id, &res) == 0) info->resolution = res.tv_sec + res.tv_nsec * 1e-9; else @@ -1071,7 +1092,7 @@ py_process_time(_Py_clock_info_t *info) if (info) { info->implementation = "getrusage(RUSAGE_SELF)"; info->monotonic = 1; - info->adjusted = 0; + info->adjustable = 0; info->resolution = 1e-6; } return PyFloat_FromDouble(total); @@ -1100,7 +1121,7 @@ py_process_time(_Py_clock_info_t *info) if (info) { info->implementation = "times()"; info->monotonic = 1; - info->adjusted = 0; + info->adjustable = 0; info->resolution = 1.0 / ticks_per_second; } return PyFloat_FromDouble(total); @@ -1124,35 +1145,12 @@ PyDoc_STRVAR(process_time_doc, Process time for profiling: sum of the kernel and user-space CPU time."); -static PyTypeObject ClockInfoType; - -PyDoc_STRVAR(ClockInfo_docstring, - "Clock information"); - -static PyStructSequence_Field ClockInfo_fields[] = { - {"implementation", "name of the underlying C function " - "used to get the clock value"}, - {"monotonic", "True if the clock cannot go backward, False otherwise"}, - {"adjusted", "True if the clock can be adjusted " - "(e.g. by a NTP daemon), False otherwise"}, - {"resolution", "resolution of the clock in seconds"}, - {NULL, NULL} -}; - -static PyStructSequence_Desc ClockInfo_desc = { - "time.clock_info", - ClockInfo_docstring, - ClockInfo_fields, - 4, -}; - static PyObject * time_get_clock_info(PyObject *self, PyObject *args) { char *name; - PyObject *obj; _Py_clock_info_t info; - PyObject *result; + PyObject *obj = NULL, *dict, *ns; if (!PyArg_ParseTuple(args, "s:get_clock_info", &name)) return NULL; @@ -1160,12 +1158,12 @@ time_get_clock_info(PyObject *self, PyObject *args) #ifdef Py_DEBUG info.implementation = NULL; info.monotonic = -1; - info.adjusted = -1; + info.adjustable = -1; info.resolution = -1.0; #else info.implementation = ""; info.monotonic = 0; - info.adjusted = 0; + info.adjustable = 0; info.resolution = 1.0; #endif @@ -1191,39 +1189,50 @@ time_get_clock_info(PyObject *self, PyObject *args) return NULL; Py_DECREF(obj); - result = PyStructSequence_New(&ClockInfoType); - if (result == NULL) + dict = PyDict_New(); + if (dict == NULL) return NULL; assert(info.implementation != NULL); obj = PyUnicode_FromString(info.implementation); if (obj == NULL) goto error; - PyStructSequence_SET_ITEM(result, 0, obj); + if (PyDict_SetItemString(dict, "implementation", obj) == -1) + goto error; + Py_CLEAR(obj); assert(info.monotonic != -1); obj = PyBool_FromLong(info.monotonic); if (obj == NULL) goto error; - PyStructSequence_SET_ITEM(result, 1, obj); + if (PyDict_SetItemString(dict, "monotonic", obj) == -1) + goto error; + Py_CLEAR(obj); - assert(info.adjusted != -1); - obj = PyBool_FromLong(info.adjusted); + assert(info.adjustable != -1); + obj = PyBool_FromLong(info.adjustable); if (obj == NULL) goto error; - PyStructSequence_SET_ITEM(result, 2, obj); + if (PyDict_SetItemString(dict, "adjustable", obj) == -1) + goto error; + Py_CLEAR(obj); assert(info.resolution > 0.0); assert(info.resolution <= 1.0); obj = PyFloat_FromDouble(info.resolution); if (obj == NULL) goto error; - PyStructSequence_SET_ITEM(result, 3, obj); + if (PyDict_SetItemString(dict, "resolution", obj) == -1) + goto error; + Py_CLEAR(obj); - return result; + ns = _PyNamespace_New(dict); + Py_DECREF(dict); + return ns; error: - Py_DECREF(result); + Py_DECREF(dict); + Py_XDECREF(obj); return NULL; } @@ -1451,11 +1460,6 @@ PyInit_time(void) PyStructSequence_InitType(&StructTimeType, &struct_time_type_desc); - /* initialize ClockInfoType */ - PyStructSequence_InitType(&ClockInfoType, &ClockInfo_desc); - Py_INCREF(&ClockInfoType); - PyModule_AddObject(m, "clock_info", (PyObject*)&ClockInfoType); - #ifdef MS_WINDOWS winver.dwOSVersionInfoSize = sizeof(winver); if (!GetVersionEx((OSVERSIONINFO*)&winver)) { @@ -1466,6 +1470,11 @@ PyInit_time(void) #endif } Py_INCREF(&StructTimeType); +#ifdef HAVE_STRUCT_TM_TM_ZONE + PyModule_AddIntConstant(m, "_STRUCT_TM_ITEMS", 11); +#else + PyModule_AddIntConstant(m, "_STRUCT_TM_ITEMS", 9); +#endif PyModule_AddObject(m, "struct_time", (PyObject*) &StructTimeType); initialized = 1; return m; @@ -1488,7 +1497,7 @@ floattime(_Py_clock_info_t *info) struct timespec res; info->implementation = "clock_gettime(CLOCK_REALTIME)"; info->monotonic = 0; - info->adjusted = 1; + info->adjustable = 1; if (clock_getres(CLOCK_REALTIME, &res) == 0) info->resolution = res.tv_sec + res.tv_nsec * 1e-9; else diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index 07627d6..fb35493 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -562,4 +562,68 @@ IllegalSurrogate: #undef STRIPPED_MASK #undef SWAB #undef LONG_PTR_MASK + + +Py_LOCAL_INLINE(void) +STRINGLIB(utf16_encode)(unsigned short *out, + const STRINGLIB_CHAR *in, + Py_ssize_t len, + int native_ordering) +{ + const STRINGLIB_CHAR *end = in + len; +#if STRINGLIB_SIZEOF_CHAR == 1 +# define SWAB2(CH) ((CH) << 8) +#else +# define SWAB2(CH) (((CH) << 8) | ((CH) >> 8)) +#endif +#if STRINGLIB_MAX_CHAR < 0x10000 + if (native_ordering) { +# if STRINGLIB_SIZEOF_CHAR == 2 + Py_MEMCPY(out, in, 2 * len); +# else + _PyUnicode_CONVERT_BYTES(STRINGLIB_CHAR, unsigned short, in, end, out); +# endif + } else { + const STRINGLIB_CHAR *unrolled_end = in + (len & ~ (Py_ssize_t) 3); + while (in < unrolled_end) { + out[0] = SWAB2(in[0]); + out[1] = SWAB2(in[1]); + out[2] = SWAB2(in[2]); + out[3] = SWAB2(in[3]); + in += 4; out += 4; + } + while (in < end) { + *out++ = SWAB2(*in); + ++in; + } + } +#else + if (native_ordering) { + while (in < end) { + Py_UCS4 ch = *in++; + if (ch < 0x10000) + *out++ = ch; + else { + out[0] = Py_UNICODE_HIGH_SURROGATE(ch); + out[1] = Py_UNICODE_LOW_SURROGATE(ch); + out += 2; + } + } + } else { + while (in < end) { + Py_UCS4 ch = *in++; + if (ch < 0x10000) + *out++ = SWAB2((Py_UCS2)ch); + else { + Py_UCS2 ch1 = Py_UNICODE_HIGH_SURROGATE(ch); + Py_UCS2 ch2 = Py_UNICODE_LOW_SURROGATE(ch); + out[0] = SWAB2(ch1); + out[1] = SWAB2(ch2); + out += 2; + } + } + } +#endif +#undef SWAB2 +} #endif /* STRINGLIB_IS_UNICODE */ diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index a1efec0..c974ffe 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5359,27 +5359,19 @@ _PyUnicode_EncodeUTF16(PyObject *str, const char *errors, int byteorder) { - int kind; - void *data; + enum PyUnicode_Kind kind; + const void *data; Py_ssize_t len; PyObject *v; - unsigned char *p; - Py_ssize_t nsize, bytesize; - Py_ssize_t i, pairs; - /* Offsets from p for storing byte pairs in the right order. */ -#ifdef BYTEORDER_IS_LITTLE_ENDIAN - int ihi = 1, ilo = 0; + unsigned short *out; + Py_ssize_t bytesize; + Py_ssize_t pairs; +#ifdef WORDS_BIGENDIAN + int native_ordering = byteorder >= 0; #else - int ihi = 0, ilo = 1; + int native_ordering = byteorder <= 0; #endif -#define STORECHAR(CH) \ - do { \ - p[ihi] = ((CH) >> 8) & 0xff; \ - p[ilo] = (CH) & 0xff; \ - p += 2; \ - } while(0) - if (!PyUnicode_Check(str)) { PyErr_BadArgument(); return NULL; @@ -5391,53 +5383,47 @@ _PyUnicode_EncodeUTF16(PyObject *str, len = PyUnicode_GET_LENGTH(str); pairs = 0; - if (kind == PyUnicode_4BYTE_KIND) - for (i = 0; i < len; i++) - if (PyUnicode_READ(kind, data, i) >= 0x10000) + if (kind == PyUnicode_4BYTE_KIND) { + const Py_UCS4 *in = (const Py_UCS4 *)data; + const Py_UCS4 *end = in + len; + while (in < end) + if (*in++ >= 0x10000) pairs++; - /* 2 * (len + pairs + (byteorder == 0)) */ - if (len > PY_SSIZE_T_MAX - pairs - (byteorder == 0)) - return PyErr_NoMemory(); - nsize = len + pairs + (byteorder == 0); - bytesize = nsize * 2; - if (bytesize / 2 != nsize) + } + if (len > PY_SSIZE_T_MAX / 2 - pairs - (byteorder == 0)) return PyErr_NoMemory(); + bytesize = (len + pairs + (byteorder == 0)) * 2; v = PyBytes_FromStringAndSize(NULL, bytesize); if (v == NULL) return NULL; - p = (unsigned char *)PyBytes_AS_STRING(v); + /* output buffer is 2-bytes aligned */ + assert(((Py_uintptr_t)PyBytes_AS_STRING(v) & 1) == 0); + out = (unsigned short *)PyBytes_AS_STRING(v); if (byteorder == 0) - STORECHAR(0xFEFF); + *out++ = 0xFEFF; if (len == 0) goto done; - if (byteorder == -1) { - /* force LE */ - ihi = 1; - ilo = 0; + switch (kind) { + case PyUnicode_1BYTE_KIND: { + ucs1lib_utf16_encode(out, (const Py_UCS1 *)data, len, native_ordering); + break; } - else if (byteorder == 1) { - /* force BE */ - ihi = 0; - ilo = 1; + case PyUnicode_2BYTE_KIND: { + ucs2lib_utf16_encode(out, (const Py_UCS2 *)data, len, native_ordering); + break; } - - for (i = 0; i < len; i++) { - Py_UCS4 ch = PyUnicode_READ(kind, data, i); - Py_UCS4 ch2 = 0; - if (ch >= 0x10000) { - ch2 = Py_UNICODE_LOW_SURROGATE(ch); - ch = Py_UNICODE_HIGH_SURROGATE(ch); - } - STORECHAR(ch); - if (ch2) - STORECHAR(ch2); + case PyUnicode_4BYTE_KIND: { + ucs4lib_utf16_encode(out, (const Py_UCS4 *)data, len, native_ordering); + break; + } + default: + assert(0); } done: return v; -#undef STORECHAR } PyObject * diff --git a/PC/VS9.0/pythoncore.vcproj b/PC/VS9.0/pythoncore.vcproj index 44a1e5f..9fb63ff 100644 --- a/PC/VS9.0/pythoncore.vcproj +++ b/PC/VS9.0/pythoncore.vcproj @@ -803,6 +803,10 @@ >
</File>
<File
+ RelativePath="..\..\Include\namespaceobject.h"
+ >
+ </File>
+ <File
RelativePath="..\..\Include\node.h"
>
</File>
@@ -1563,6 +1567,10 @@ >
</File>
<File
+ RelativePath="..\..\Objects\namespaceobject.c"
+ >
+ </File>
+ <File
RelativePath="..\..\Objects\object.c"
>
</File>
diff --git a/Python/pytime.c b/Python/pytime.c index eb5685b..beeab87 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -44,10 +44,7 @@ pygettimeofday(_PyTime_timeval *tp, _Py_clock_info_t *info) (void) GetSystemTimeAdjustment(&timeAdjustment, &timeIncrement, &isTimeAdjustmentDisabled); info->resolution = timeIncrement * 1e-7; - if (isTimeAdjustmentDisabled) - info->adjusted = 0; - else - info->adjusted = 1; + info->adjustable = 1; } #else /* There are three ways to get the time: @@ -71,7 +68,7 @@ pygettimeofday(_PyTime_timeval *tp, _Py_clock_info_t *info) info->implementation = "gettimeofday()"; info->resolution = 1e-6; info->monotonic = 0; - info->adjusted = 1; + info->adjustable = 1; } return; } @@ -87,7 +84,7 @@ pygettimeofday(_PyTime_timeval *tp, _Py_clock_info_t *info) info->implementation = "ftime()"; info->resolution = 1e-3; info->monotonic = 0; - info->adjusted = 1; + info->adjustable = 1; } } #else /* !HAVE_FTIME */ @@ -97,7 +94,7 @@ pygettimeofday(_PyTime_timeval *tp, _Py_clock_info_t *info) info->implementation = "time()"; info->resolution = 1.0; info->monotonic = 0; - info->adjusted = 1; + info->adjustable = 1; } #endif /* !HAVE_FTIME */ |